From 85304eadd1f6cc91dc5436fa3bc72528de4cdbcc Mon Sep 17 00:00:00 2001 From: Stevan Freeborn <65925598+StevanFreeborn@users.noreply.github.com> Date: Tue, 12 Aug 2025 00:17:37 -0500 Subject: [PATCH] feat: trying to create clips...probs a waste of time grrrrrrr --- .../Commands/DefaultCommand.cs | 63 ++++++++++++++++++- .../Hosting/HostBuilderExtensions.cs | 2 +- src/StreamShorts.Console/Program.cs | 16 ++++- .../Analysis/Gemini/GeminiAnalyzer.cs | 2 +- .../{ShortClip.cs => ShortCandidate.cs} | 4 +- .../Analysis/TranscriptAnalysis.cs | 4 +- .../Media/FFMpegService.cs | 17 +++++ .../Media/IAudioService.cs | 4 +- .../Media/IVideoService.cs | 10 +++ .../Media/Video/IShortsCreator.cs | 15 +++++ .../Media/Video/ShortClip.cs | 11 ++++ .../Media/Video/ShortsCreator.cs | 42 ++++++++++++- 12 files changed, 177 insertions(+), 13 deletions(-) rename src/StreamShorts.Library/Analysis/{ShortClip.cs => ShortCandidate.cs} (82%) create mode 100644 src/StreamShorts.Library/Media/Video/ShortClip.cs diff --git a/src/StreamShorts.Console/Commands/DefaultCommand.cs b/src/StreamShorts.Console/Commands/DefaultCommand.cs index 62a921a..8c6157c 100644 --- a/src/StreamShorts.Console/Commands/DefaultCommand.cs +++ b/src/StreamShorts.Console/Commands/DefaultCommand.cs @@ -1,3 +1,8 @@ +using System.Globalization; +using System.Text.Json; + +using StreamShorts.Library.Media.Video; + namespace StreamShorts.Console.Commands; internal sealed class DefaultCommand( @@ -5,14 +10,24 @@ internal sealed class DefaultCommand( IAnsiConsole console, IAudioExtractor audioExtractor, ITranscriber transcriber, - ITranscriptAnalyzer transcriptAnalyzer + ITranscriptAnalyzer transcriptAnalyzer, + IShortsCreator shortsCreator, + TimeProvider timeProvider ) : AsyncCommand { + private readonly JsonSerializerOptions _jsonSerializerOptions = new() + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); private readonly IAnsiConsole _console = console ?? throw new ArgumentNullException(nameof(console)); private readonly IAudioExtractor _audioExtractor = audioExtractor ?? throw new ArgumentNullException(nameof(audioExtractor)); private readonly ITranscriber _transcriber = transcriber ?? throw new ArgumentNullException(nameof(transcriber)); private readonly ITranscriptAnalyzer _transcriptAnalyzer = transcriptAnalyzer ?? throw new ArgumentNullException(nameof(transcriptAnalyzer)); + private readonly IShortsCreator _shortsCreator = shortsCreator ?? throw new ArgumentNullException(nameof(shortsCreator)); + private readonly TimeProvider _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); internal class Settings : CommandSettings { @@ -60,7 +75,7 @@ internal sealed class DefaultCommand( if (audioStream is null) { _console.MarkupLine("[red]Failed[/] to extract audio from the stream."); - return 1; + return (int)ExitCode.FailedToExtractAudio; } _console.MarkupLine($"[blue]Audio extracted[/] [green]successfully![/]"); @@ -90,6 +105,50 @@ internal sealed class DefaultCommand( analysis = await _transcriptAnalyzer.AnalyzeAsync(transcriptionSegments); }); + if (analysis is null) + { + _console.MarkupLine("[red]Failed[/] to analyze transcript."); + return (int)ExitCode.FailedToAnalyzeTranscript; + } + + _console.MarkupLine($"[blue]Transcript analysis completed[/] [green]successfully![/]"); + + var now = _timeProvider.GetUtcNow(); + var inputFileName = _fileSystem.Path.GetFileNameWithoutExtension(settings.Stream); + var outputDirectoryPath = _fileSystem.Path.Combine( + AppContext.BaseDirectory, + $"{now:yyyy_MM_dd}_{inputFileName}" + ); + + var outputDirectory = _fileSystem.Directory.CreateDirectory(outputDirectoryPath); + + await _fileSystem.File.WriteAllTextAsync( + _fileSystem.Path.Combine(outputDirectoryPath, "analysis.json"), + JsonSerializer.Serialize(analysis, _jsonSerializerOptions) + ); + + await _console.Status() + .Spinner(Spinner.Known.Dots) + .StartAsync("Creating shorts...", async ctx => + { + await foreach (var clip in _shortsCreator.CreateShortsAsync(analysis, videoStream)) + { + ctx.Status($"Creating short: {clip.Candidate.Title.EscapeMarkup()}"); + + var safeFileName = string.Concat(clip.Candidate.Title.Split(_fileSystem.Path.GetInvalidFileNameChars())); + var filePath = _fileSystem.Path.Combine(outputDirectoryPath, $"{safeFileName}.webm"); + var fileStream = _fileSystem.File.OpenWrite(filePath); + } + }); + + _console.MarkupLine($"[blue]Shorts created[/] [green]successfully![/]"); + return 0; } + + internal enum ExitCode + { + FailedToExtractAudio, + FailedToAnalyzeTranscript, + } } \ No newline at end of file diff --git a/src/StreamShorts.Console/Hosting/HostBuilderExtensions.cs b/src/StreamShorts.Console/Hosting/HostBuilderExtensions.cs index 4b4bdd6..374372e 100644 --- a/src/StreamShorts.Console/Hosting/HostBuilderExtensions.cs +++ b/src/StreamShorts.Console/Hosting/HostBuilderExtensions.cs @@ -11,7 +11,7 @@ internal static class HostBuilderExtensions c.SetExceptionHandler(static (ex, resolver) => { var console = resolver?.Resolve(typeof(IAnsiConsole)) as IAnsiConsole; - console?.WriteLine($"[red]An error occurred while executing the command:[/]"); + console?.MarkupLine($"[red]An error occurred while executing the command:[/]"); console?.WriteException(ex, ExceptionFormats.ShortenEverything); }) ); diff --git a/src/StreamShorts.Console/Program.cs b/src/StreamShorts.Console/Program.cs index 991ddfb..95a6bd3 100644 --- a/src/StreamShorts.Console/Program.cs +++ b/src/StreamShorts.Console/Program.cs @@ -1,4 +1,8 @@ -Log.Logger = new LoggerConfiguration() +using Microsoft.Extensions.Configuration; + +using StreamShorts.Library.Media.Video; + +Log.Logger = new LoggerConfiguration() .WriteTo.File( formatter: new CompactJsonFormatter(), path: Path.Combine(AppContext.BaseDirectory, "logs", "log.jsonl"), @@ -18,11 +22,19 @@ try .ConfigureLogging(static l => l.ClearProviders()) .ConfigureServices(static (_, services) => { + services.AddHttpClient(); services.AddSingleton(AnsiConsole.Console); services.AddSingleton(); + services.AddSingleton(TimeProvider.System); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(sp => + { + var clientFactory = sp.GetRequiredService(); + + return new GeminiAnalyzer(clientFactory, ""); + }); + services.AddSingleton(); }) .BuildApp() .RunAsync(args); diff --git a/src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs b/src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs index d5f23ce..2e56260 100644 --- a/src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs +++ b/src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs @@ -63,7 +63,7 @@ public sealed class GeminiAnalyzer( .Parts?.FirstOrDefault()? .Text; - var clips = JsonSerializer.Deserialize>(candidatesText ?? string.Empty); + var clips = JsonSerializer.Deserialize>(candidatesText ?? string.Empty); return new TranscriptAnalysis(clips ?? []); } } diff --git a/src/StreamShorts.Library/Analysis/ShortClip.cs b/src/StreamShorts.Library/Analysis/ShortCandidate.cs similarity index 82% rename from src/StreamShorts.Library/Analysis/ShortClip.cs rename to src/StreamShorts.Library/Analysis/ShortCandidate.cs index 3f8901b..42433cc 100644 --- a/src/StreamShorts.Library/Analysis/ShortClip.cs +++ b/src/StreamShorts.Library/Analysis/ShortCandidate.cs @@ -3,9 +3,9 @@ using System.Text.Json.Serialization; namespace StreamShorts.Library.Analysis; /// -/// Represents a short clip derived from a transcript +/// Represents a short candidate derived from a transcript. /// -public record ShortClip( +public record ShortCandidate( [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("description")] diff --git a/src/StreamShorts.Library/Analysis/TranscriptAnalysis.cs b/src/StreamShorts.Library/Analysis/TranscriptAnalysis.cs index 87172c2..5900bdd 100644 --- a/src/StreamShorts.Library/Analysis/TranscriptAnalysis.cs +++ b/src/StreamShorts.Library/Analysis/TranscriptAnalysis.cs @@ -3,10 +3,10 @@ namespace StreamShorts.Library.Analysis; /// /// Represents the analysis of a transcript, containing short clips derived from the transcript. /// -public sealed class TranscriptAnalysis(IEnumerable shortClips) +public sealed class TranscriptAnalysis(IEnumerable candidates) { /// /// Gets the short clips derived from the transcript. /// - public IEnumerable ShortClips { get; init; } = shortClips; + public IEnumerable Candidates { get; init; } = candidates; } \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/FFMpegService.cs b/src/StreamShorts.Library/Media/FFMpegService.cs index b2e9a7d..e136c5c 100644 --- a/src/StreamShorts.Library/Media/FFMpegService.cs +++ b/src/StreamShorts.Library/Media/FFMpegService.cs @@ -1,3 +1,4 @@ + using FFMpegCore; using FFMpegCore.Enums; using FFMpegCore.Pipes; @@ -21,4 +22,20 @@ internal sealed class FFMpegService : IVideoService .ProcessAsynchronously() .ConfigureAwait(false); } + + public async Task ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null) + { + var startTime = start - (buffer ?? TimeSpan.Zero); + var endTime = end + (buffer ?? TimeSpan.Zero); + var outputStream = new MemoryStream(); + + await FFMpegArguments + .FromPipeInput(new StreamPipeSource(video), o => o.Seek(startTime).EndSeek(endTime)) + .OutputToPipe(new StreamPipeSink(outputStream), o => o.CopyChannel().ForceFormat("webm")) + .ProcessAsynchronously() + .ConfigureAwait(false); + + outputStream.Position = 0; + return outputStream; + } } \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/IAudioService.cs b/src/StreamShorts.Library/Media/IAudioService.cs index 5410c57..a41d427 100644 --- a/src/StreamShorts.Library/Media/IAudioService.cs +++ b/src/StreamShorts.Library/Media/IAudioService.cs @@ -18,7 +18,7 @@ internal interface IAudioService /// /// Gets the number of segments in a WAV stream based on the specified segment duration. /// - /// param name="wavStream">The input WAV stream. + /// The input WAV stream. /// The duration of each segment. /// The number of segments. /// Thrown when the WAV stream is null. @@ -29,7 +29,7 @@ internal interface IAudioService /// /// Gets a segment of a WAV stream based on the specified segment number and duration. /// - /// param name="wavStream">The input WAV stream. + /// The input WAV stream. /// The segment number to retrieve. /// The duration of each segment. /// The segment stream. diff --git a/src/StreamShorts.Library/Media/IVideoService.cs b/src/StreamShorts.Library/Media/IVideoService.cs index 70c3d1b..1c48b89 100644 --- a/src/StreamShorts.Library/Media/IVideoService.cs +++ b/src/StreamShorts.Library/Media/IVideoService.cs @@ -12,4 +12,14 @@ internal interface IVideoService /// The output audio stream. /// A task that represents the asynchronous operation. The task result indicates whether the extraction was successful. Task ExtractAudioFromVideoAsync(Stream video, Stream audio); + + /// + /// Extracts a clip from a video stream. + /// + /// The input video stream. + /// The start time of the clip. + /// The end time of the clip. + /// The duration of the buffer to include before the start time and after the end time. + /// A task that represents the asynchronous operation. The task result contains the extracted clip as a stream. + Task ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null); } \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Video/IShortsCreator.cs b/src/StreamShorts.Library/Media/Video/IShortsCreator.cs index 045423a..7696366 100644 --- a/src/StreamShorts.Library/Media/Video/IShortsCreator.cs +++ b/src/StreamShorts.Library/Media/Video/IShortsCreator.cs @@ -1,5 +1,20 @@ +using StreamShorts.Library.Analysis; + namespace StreamShorts.Library.Media.Video; +/// +/// Represents a service that creates video shorts. +/// public interface IShortsCreator { + /// + /// Creates video shorts from the provided transcript analysis and video stream. + /// + /// The transcript analysis. + /// The video stream. + /// An optional buffer duration to include before the start time and after the end time of each short. + /// An asynchronous enumerable of . + /// Thrown when is null. + /// Thrown when is null. + public IAsyncEnumerable CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null); } \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Video/ShortClip.cs b/src/StreamShorts.Library/Media/Video/ShortClip.cs new file mode 100644 index 0000000..f521248 --- /dev/null +++ b/src/StreamShorts.Library/Media/Video/ShortClip.cs @@ -0,0 +1,11 @@ +using StreamShorts.Library.Analysis; + +namespace StreamShorts.Library.Media.Video; + +/// +/// Represents a short video clip. +/// +public record ShortClip( + ShortCandidate Candidate, + Stream Segment +); \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Video/ShortsCreator.cs b/src/StreamShorts.Library/Media/Video/ShortsCreator.cs index cff8875..55bc110 100644 --- a/src/StreamShorts.Library/Media/Video/ShortsCreator.cs +++ b/src/StreamShorts.Library/Media/Video/ShortsCreator.cs @@ -1,5 +1,45 @@ +using StreamShorts.Library.Analysis; + namespace StreamShorts.Library.Media.Video; +/// +/// Represents a service that creates video shorts. +/// +/// public class ShortsCreator : IShortsCreator { -} + private readonly IVideoService _videoService = new FFMpegService(); + + public ShortsCreator() + { + } + + internal ShortsCreator(IVideoService videoService) + { + _videoService = videoService ?? throw new ArgumentNullException(nameof(videoService), $"{nameof(videoService)} cannot be null"); + } + + public async IAsyncEnumerable CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null) + { + if (analysis is null) + { + throw new ArgumentNullException(nameof(analysis), $"{nameof(analysis)} cannot be null"); + } + + if (video is null) + { + throw new ArgumentNullException(nameof(video), $"{nameof(video)} cannot be null"); + } + + foreach (var candidate in analysis.Candidates) + { + var clip = await _videoService.ExtractClipFromVideoAsync(video, candidate.StartTime, candidate.EndTime, buffer) + .ConfigureAwait(false); + + yield return new ShortClip( + candidate, + clip + ); + } + } +} \ No newline at end of file