From d470574b99c30a2fa6270a1edf20e289c8640081 Mon Sep 17 00:00:00 2001 From: Stevan Freeborn <65925598+StevanFreeborn@users.noreply.github.com> Date: Tue, 19 Aug 2025 12:43:34 -0500 Subject: [PATCH] feat: complete implementation using gemini as analyzer --- .../Commands/DefaultCommand.cs | 27 +++--- src/StreamShorts.Console/Program.cs | 15 +++- .../Analysis/Gemini/GeminiAnalyzer.cs | 89 ++++++++++--------- .../Analysis/ITranscriptAnalyzer.cs | 3 +- .../Analysis/Prompts/DefaultAnalysisPrompt.cs | 3 +- .../Media/FFMpegService.cs | 24 +++-- .../Media/IVideoService.cs | 23 +++-- .../Media/Video/IShortsCreator.cs | 17 ++-- .../Media/Video/ShortClip.cs | 11 --- .../Media/Video/ShortsCreator.cs | 27 +++--- .../FailedAudioTranscriptionException.cs | 31 ------- .../Transcription/TranscriptionSegment.cs | 8 +- 12 files changed, 135 insertions(+), 143 deletions(-) delete mode 100644 src/StreamShorts.Library/Media/Video/ShortClip.cs delete mode 100644 src/StreamShorts.Library/Transcription/FailedAudioTranscriptionException.cs diff --git a/src/StreamShorts.Console/Commands/DefaultCommand.cs b/src/StreamShorts.Console/Commands/DefaultCommand.cs index 8c6157c..f62528a 100644 --- a/src/StreamShorts.Console/Commands/DefaultCommand.cs +++ b/src/StreamShorts.Console/Commands/DefaultCommand.cs @@ -1,4 +1,3 @@ -using System.Globalization; using System.Text.Json; using StreamShorts.Library.Media.Video; @@ -67,7 +66,7 @@ internal sealed class DefaultCommand( await _console.Status() .Spinner(Spinner.Known.Dots) - .StartAsync("Extracting audio...", async ctx => + .StartAsync("Extracting audio...", async _ => { audioStream = await _audioExtractor.ExtractMp3FromMp4Async(videoStream); }); @@ -96,11 +95,16 @@ internal sealed class DefaultCommand( _console.MarkupLine($"[blue]Transcription completed[/] [green]successfully![/]"); + await _fileSystem.File.WriteAllTextAsync( + _fileSystem.Path.Combine(AppContext.BaseDirectory, "transcription.txt"), + string.Join(Environment.NewLine, transcriptionSegments) + ); + TranscriptAnalysis? analysis = null; await _console.Status() .Spinner(Spinner.Known.Dots) - .StartAsync("Analyzing transcript...", async ctx => + .StartAsync("Analyzing transcript...", async _ => { analysis = await _transcriptAnalyzer.AnalyzeAsync(transcriptionSegments); }); @@ -117,10 +121,10 @@ internal sealed class DefaultCommand( var inputFileName = _fileSystem.Path.GetFileNameWithoutExtension(settings.Stream); var outputDirectoryPath = _fileSystem.Path.Combine( AppContext.BaseDirectory, - $"{now:yyyy_MM_dd}_{inputFileName}" + $"{now:yyyy_MM_dd_HH_mm_ss}_{inputFileName}" ); - var outputDirectory = _fileSystem.Directory.CreateDirectory(outputDirectoryPath); + _fileSystem.Directory.CreateDirectory(outputDirectoryPath); await _fileSystem.File.WriteAllTextAsync( _fileSystem.Path.Combine(outputDirectoryPath, "analysis.json"), @@ -131,13 +135,12 @@ internal sealed class DefaultCommand( .Spinner(Spinner.Known.Dots) .StartAsync("Creating shorts...", async ctx => { - await foreach (var clip in _shortsCreator.CreateShortsAsync(analysis, videoStream)) + foreach (var candidate in analysis.Candidates) { - ctx.Status($"Creating short: {clip.Candidate.Title.EscapeMarkup()}"); - - var safeFileName = string.Concat(clip.Candidate.Title.Split(_fileSystem.Path.GetInvalidFileNameChars())); - var filePath = _fileSystem.Path.Combine(outputDirectoryPath, $"{safeFileName}.webm"); - var fileStream = _fileSystem.File.OpenWrite(filePath); + ctx.Status($"Creating short: {candidate.Title.EscapeMarkup()}"); + var safeFileName = string.Concat(candidate.Title.Split(_fileSystem.Path.GetInvalidFileNameChars())); + var candidatePath = _fileSystem.Path.Combine(outputDirectoryPath, $"{safeFileName}.mp4"); + await _shortsCreator.CreateShortAsync(settings.Stream, candidate, candidatePath); } }); @@ -146,7 +149,7 @@ internal sealed class DefaultCommand( return 0; } - internal enum ExitCode + private enum ExitCode { FailedToExtractAudio, FailedToAnalyzeTranscript, diff --git a/src/StreamShorts.Console/Program.cs b/src/StreamShorts.Console/Program.cs index 95a6bd3..59d69cf 100644 --- a/src/StreamShorts.Console/Program.cs +++ b/src/StreamShorts.Console/Program.cs @@ -20,6 +20,7 @@ try await Host.CreateDefaultBuilder(args) .ConfigureLogging(static l => l.ClearProviders()) + .ConfigureHostConfiguration(static config => config.AddJsonFile("appsettings.json")) .ConfigureServices(static (_, services) => { services.AddHttpClient(); @@ -30,9 +31,21 @@ try services.AddSingleton(); services.AddSingleton(sp => { + const string modelOptionName = "Model"; + const string keyOptionName = "ApiKey"; + var config = sp.GetRequiredService(); + var geminiSection = config.GetSection("Gemini"); + var key = geminiSection[keyOptionName]; + var model = geminiSection[modelOptionName]; + + if (string.IsNullOrWhiteSpace(key)) + { + throw new InvalidOperationException($"{keyOptionName} is not configured in appsettings.json."); + } + var clientFactory = sp.GetRequiredService(); - return new GeminiAnalyzer(clientFactory, ""); + return new GeminiAnalyzer(clientFactory, key, model); }); services.AddSingleton(); }) diff --git a/src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs b/src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs index 2e56260..1d7b8bc 100644 --- a/src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs +++ b/src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs @@ -12,61 +12,68 @@ namespace StreamShorts.Library.Analysis.Gemini; /// public sealed class GeminiAnalyzer( IHttpClientFactory httpClientFactory, - string apiKey - ) : ITranscriptAnalyzer + string apiKey, + string? model = null +) : ITranscriptAnalyzer { - private readonly IHttpClientFactory _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + private readonly IHttpClientFactory _httpClientFactory = + httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + private readonly string _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); + private readonly string _model = model ?? "gemini-2.5-flash-lite"; private readonly IAnalysisPrompt _prompt = new DefaultAnalysisPrompt(); public GeminiAnalyzer( IHttpClientFactory httpClientFactory, string apiKey, - IAnalysisPrompt prompt - ) : this(httpClientFactory, apiKey) + IAnalysisPrompt prompt, + string? model = null + ) : this(httpClientFactory, apiKey, model) { _prompt = prompt ?? throw new ArgumentNullException(nameof(prompt)); } public async Task AnalyzeAsync(IEnumerable segments) { - using var client = _httpClientFactory.CreateClient(); - client.Timeout = TimeSpan.FromMinutes(5); - - var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={_apiKey}"; - var generateContentRequest = new GenerateContentRequest( - [ - new Content( - Role: "user", - Parts:[ new Part(Text: _prompt.GetPrompt(segments)) ] - ) - ], - new GenerationConfig(ResponseMimeType: "application/json") - ); - using var requestContent = new StringContent( - JsonSerializer.Serialize(generateContentRequest), - Encoding.UTF8, - "application/json" - ); - using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl) + try { - Content = requestContent - }; + using var client = _httpClientFactory.CreateClient(); + client.Timeout = TimeSpan.FromMinutes(5); - var response = await client.SendAsync(request).ConfigureAwait(false); - var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - var responseJson = JsonSerializer.Deserialize(responseContent); - var candidatesText = responseJson? - .Candidates? - .FirstOrDefault()? - .Content - .Parts?.FirstOrDefault()? - .Text; + var requestUrl = + $"https://generativelanguage.googleapis.com/v1beta/models/{_model}:generateContent?key={_apiKey}"; + var generateContentRequest = new GenerateContentRequest( + [ + new Content( + Role: "user", + Parts: [new Part(Text: _prompt.GetPrompt(segments))] + ) + ], + new GenerationConfig(ResponseMimeType: "application/json") + ); + using var requestContent = new StringContent( + JsonSerializer.Serialize(generateContentRequest), + Encoding.UTF8, + "application/json" + ); + using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl) { Content = requestContent }; - var clips = JsonSerializer.Deserialize>(candidatesText ?? string.Empty); - return new TranscriptAnalysis(clips ?? []); + var response = await client.SendAsync(request).ConfigureAwait(false); + var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var responseJson = JsonSerializer.Deserialize(responseContent); + var candidatesText = responseJson? + .Candidates? + .FirstOrDefault()? + .Content + .Parts?.FirstOrDefault()? + .Text; + + var clips = JsonSerializer.Deserialize>(candidatesText ?? string.Empty); + return new TranscriptAnalysis(clips ?? []); + } + catch (Exception e) + { + throw new FailedTranscriptAnalysisException("Failed to analyze transcript segments using Gemini.", e); + } } -} - - - +} \ No newline at end of file diff --git a/src/StreamShorts.Library/Analysis/ITranscriptAnalyzer.cs b/src/StreamShorts.Library/Analysis/ITranscriptAnalyzer.cs index 3e51ac5..9713ee2 100644 --- a/src/StreamShorts.Library/Analysis/ITranscriptAnalyzer.cs +++ b/src/StreamShorts.Library/Analysis/ITranscriptAnalyzer.cs @@ -11,6 +11,7 @@ public interface ITranscriptAnalyzer /// Analyzes the provided transcript segments and generates a transcript analysis result. /// /// The transcript segments to analyze. - /// A task that represents the asynchronous operation. The task result contains the containing the short clips derived from the transcript. + /// A task that represents the asynchronous operation. The task result contains the containing the short clips derived from the transcript. + /// Thrown when the analysis fails due to an error. Task AnalyzeAsync(IEnumerable segments); } \ No newline at end of file diff --git a/src/StreamShorts.Library/Analysis/Prompts/DefaultAnalysisPrompt.cs b/src/StreamShorts.Library/Analysis/Prompts/DefaultAnalysisPrompt.cs index 1d174a3..3d6190b 100644 --- a/src/StreamShorts.Library/Analysis/Prompts/DefaultAnalysisPrompt.cs +++ b/src/StreamShorts.Library/Analysis/Prompts/DefaultAnalysisPrompt.cs @@ -42,8 +42,9 @@ internal sealed class DefaultAnalysisPrompt : IAnalysisPrompt {0} "); - public string GetPrompt(IEnumerable transcript) + public string GetPrompt(IEnumerable segments) { + var transcript = string.Join(Environment.NewLine, segments); return string.Format(CultureInfo.InvariantCulture, Prompt, transcript); } } \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/FFMpegService.cs b/src/StreamShorts.Library/Media/FFMpegService.cs index e136c5c..3f74a72 100644 --- a/src/StreamShorts.Library/Media/FFMpegService.cs +++ b/src/StreamShorts.Library/Media/FFMpegService.cs @@ -1,4 +1,3 @@ - using FFMpegCore; using FFMpegCore.Enums; using FFMpegCore.Pipes; @@ -23,19 +22,18 @@ internal sealed class FFMpegService : IVideoService .ConfigureAwait(false); } - public async Task ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null) + public async Task CreateClipFromVideoAsync( + string sourcePath, + string destinationPath, + TimeSpan startTime, + TimeSpan endTime, + TimeSpan? buffer = null + ) { - var startTime = start - (buffer ?? TimeSpan.Zero); - var endTime = end + (buffer ?? TimeSpan.Zero); - var outputStream = new MemoryStream(); - - await FFMpegArguments - .FromPipeInput(new StreamPipeSource(video), o => o.Seek(startTime).EndSeek(endTime)) - .OutputToPipe(new StreamPipeSink(outputStream), o => o.CopyChannel().ForceFormat("webm")) - .ProcessAsynchronously() + var start = startTime - (buffer ?? TimeSpan.Zero); + var end = endTime + (buffer ?? TimeSpan.Zero); + + await FFMpeg.SubVideoAsync(sourcePath, destinationPath, start, end) .ConfigureAwait(false); - - outputStream.Position = 0; - return outputStream; } } \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/IVideoService.cs b/src/StreamShorts.Library/Media/IVideoService.cs index 1c48b89..2818580 100644 --- a/src/StreamShorts.Library/Media/IVideoService.cs +++ b/src/StreamShorts.Library/Media/IVideoService.cs @@ -12,14 +12,21 @@ internal interface IVideoService /// The output audio stream. /// A task that represents the asynchronous operation. The task result indicates whether the extraction was successful. Task ExtractAudioFromVideoAsync(Stream video, Stream audio); - + /// - /// Extracts a clip from a video stream. + /// Creates a clip from a video file based on the specified start and end times. /// - /// The input video stream. - /// The start time of the clip. - /// The end time of the clip. - /// The duration of the buffer to include before the start time and after the end time. - /// A task that represents the asynchronous operation. The task result contains the extracted clip as a stream. - Task ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null); + /// The path to the source video file. + /// The path where the created clip will be saved. + /// The start time of the clip. + /// The end time of the clip. + /// An optional buffer time to include before and after the clip segment. + /// A task that represents the asynchronous operation. + Task CreateClipFromVideoAsync( + string sourcePath, + string destinationPath, + TimeSpan startTime, + TimeSpan endTime, + TimeSpan? buffer = null + ); } \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Video/IShortsCreator.cs b/src/StreamShorts.Library/Media/Video/IShortsCreator.cs index 7696366..011f80a 100644 --- a/src/StreamShorts.Library/Media/Video/IShortsCreator.cs +++ b/src/StreamShorts.Library/Media/Video/IShortsCreator.cs @@ -8,13 +8,14 @@ namespace StreamShorts.Library.Media.Video; public interface IShortsCreator { /// - /// Creates video shorts from the provided transcript analysis and video stream. + /// Creates a video short from the specified source video file based on the provided candidate details. /// - /// The transcript analysis. - /// The video stream. - /// An optional buffer duration to include before the start time and after the end time of each short. - /// An asynchronous enumerable of . - /// Thrown when is null. - /// Thrown when is null. - public IAsyncEnumerable CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null); + /// The path to the source video file. + /// The details of the short candidate. + /// The path where the created short will be saved. + /// An optional buffer time to include before and after the short segment. + /// A task that represents the asynchronous operation. + /// Thrown when or is null or whitespace. + /// Thrown when is null. + public Task CreateShortAsync(string sourcePath, ShortCandidate candidate, string destinationPath, TimeSpan? buffer = null); } \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Video/ShortClip.cs b/src/StreamShorts.Library/Media/Video/ShortClip.cs deleted file mode 100644 index f521248..0000000 --- a/src/StreamShorts.Library/Media/Video/ShortClip.cs +++ /dev/null @@ -1,11 +0,0 @@ -using StreamShorts.Library.Analysis; - -namespace StreamShorts.Library.Media.Video; - -/// -/// Represents a short video clip. -/// -public record ShortClip( - ShortCandidate Candidate, - Stream Segment -); \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Video/ShortsCreator.cs b/src/StreamShorts.Library/Media/Video/ShortsCreator.cs index 55bc110..04d7240 100644 --- a/src/StreamShorts.Library/Media/Video/ShortsCreator.cs +++ b/src/StreamShorts.Library/Media/Video/ShortsCreator.cs @@ -19,27 +19,24 @@ public class ShortsCreator : IShortsCreator _videoService = videoService ?? throw new ArgumentNullException(nameof(videoService), $"{nameof(videoService)} cannot be null"); } - public async IAsyncEnumerable CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null) + public async Task CreateShortAsync(string sourcePath, ShortCandidate candidate, string destinationPath, TimeSpan? buffer = null) { - if (analysis is null) + if (string.IsNullOrWhiteSpace(sourcePath)) { - throw new ArgumentNullException(nameof(analysis), $"{nameof(analysis)} cannot be null"); + throw new ArgumentNullException(nameof(sourcePath), $"{nameof(sourcePath)} cannot be null or whitespace"); } - - if (video is null) + + if (candidate is null) { - throw new ArgumentNullException(nameof(video), $"{nameof(video)} cannot be null"); + throw new ArgumentNullException(nameof(candidate), $"{nameof(candidate)} cannot be null"); } - - foreach (var candidate in analysis.Candidates) + + if (string.IsNullOrWhiteSpace(destinationPath)) { - var clip = await _videoService.ExtractClipFromVideoAsync(video, candidate.StartTime, candidate.EndTime, buffer) - .ConfigureAwait(false); - - yield return new ShortClip( - candidate, - clip - ); + throw new ArgumentNullException(nameof(destinationPath), $"{nameof(destinationPath)} cannot be null or whitespace"); } + + await _videoService.CreateClipFromVideoAsync(sourcePath, destinationPath, candidate.StartTime, candidate.EndTime, buffer) + .ConfigureAwait(false); } } \ No newline at end of file diff --git a/src/StreamShorts.Library/Transcription/FailedAudioTranscriptionException.cs b/src/StreamShorts.Library/Transcription/FailedAudioTranscriptionException.cs deleted file mode 100644 index 8d7a3d1..0000000 --- a/src/StreamShorts.Library/Transcription/FailedAudioTranscriptionException.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace StreamShorts.Library.Transcription; - -/// -/// Represents an error that occurs when audio transcription fails. -/// -public sealed class FailedAudioTranscriptionException : Exception -{ - /// - /// Initializes a new instance of the class with no parameters. - /// - public FailedAudioTranscriptionException() - { - } - - /// - /// Initializes a new instance of the class with a specified error message. - /// - /// The error message that explains the reason for the exception. - public FailedAudioTranscriptionException(string message) : base(message) - { - } - - /// - /// Initializes a new instance of the class with a specified error message and a reference to the inner exception that is the cause of this exception. - /// - /// The error message that explains the reason for the exception. - /// The exception that is the cause of the current exception. - public FailedAudioTranscriptionException(string message, Exception innerException) : base(message, innerException) - { - } -} \ No newline at end of file diff --git a/src/StreamShorts.Library/Transcription/TranscriptionSegment.cs b/src/StreamShorts.Library/Transcription/TranscriptionSegment.cs index 5ea5020..cce9d32 100644 --- a/src/StreamShorts.Library/Transcription/TranscriptionSegment.cs +++ b/src/StreamShorts.Library/Transcription/TranscriptionSegment.cs @@ -10,4 +10,10 @@ public sealed record TranscriptionSegment( TimeSpan StartTime, TimeSpan EndTime, string Text -); \ No newline at end of file +) +{ + public override string ToString() + { + return $"[{StartTime:hh\\:mm\\:ss} - {EndTime:hh\\:mm\\:ss}]: {Text}"; + } +} \ No newline at end of file