diff --git a/.editorconfig b/.editorconfig index 0adf1e7..b4b5c8a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -26,6 +26,7 @@ insert_final_newline = false # Diagnostic severity preferences dotnet_diagnostic.IDE0058.severity = none dotnet_diagnostic.IDE0100.severity = none +dotnet_diagnostic.CA1848.severity = none # Organize usings dotnet_separate_import_directive_groups = true diff --git a/src/Directory.packages.props b/src/Directory.packages.props index 0f718b5..a2138ae 100644 --- a/src/Directory.packages.props +++ b/src/Directory.packages.props @@ -7,6 +7,7 @@ + diff --git a/src/StreamShorts.Console/Commands/DefaultCommand.cs b/src/StreamShorts.Console/Commands/DefaultCommand.cs index 60ae4fe..152dc91 100644 --- a/src/StreamShorts.Console/Commands/DefaultCommand.cs +++ b/src/StreamShorts.Console/Commands/DefaultCommand.cs @@ -2,11 +2,13 @@ namespace StreamShorts.Console.Commands; internal class DefaultCommand( IFileSystem fileSystem, - IAnsiConsole console + IAnsiConsole console, + IAudioExtractor audioExtractor ) : AsyncCommand { private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); private readonly IAnsiConsole _console = console ?? throw new ArgumentNullException(nameof(console)); + private readonly IAudioExtractor _audioExtractor = audioExtractor ?? throw new ArgumentNullException(nameof(audioExtractor)); internal class Settings : CommandSettings { @@ -37,9 +39,28 @@ internal class DefaultCommand( return base.Validate(context, settings); } - public override Task ExecuteAsync(CommandContext context, Settings settings) + public override async Task ExecuteAsync(CommandContext context, Settings settings) { - _console.Write($"[green]Processing stream:[/] {settings.Stream}"); - return Task.FromResult(0); + _console.MarkupLine($"[blue]Processing stream:[/] {settings.Stream}"); + var videoStream = _fileSystem.File.OpenRead(settings.Stream); + + Stream? audioStream = null; + + await _console.Status() + .Spinner(Spinner.Known.Dots) + .StartAsync("Extracting audio...", async ctx => + { + audioStream = await _audioExtractor.ExtractMp3FromMp4Async(videoStream).ConfigureAwait(false); + }); + + if (audioStream is null) + { + _console.MarkupLine("[red]Failed[/] to extract audio from the stream."); + return 1; + } + + _console.MarkupLine($"[blue]Audio extracted[/] [green]successfully![/]"); + + return 0; } } \ No newline at end of file diff --git a/src/StreamShorts.Console/Program.cs b/src/StreamShorts.Console/Program.cs index 53065ed..8d518aa 100644 --- a/src/StreamShorts.Console/Program.cs +++ b/src/StreamShorts.Console/Program.cs @@ -10,6 +10,7 @@ using FFMpegCore.Pipes; using NAudio.Wave; + using Whisper.net; using Whisper.net.Ggml; @@ -35,6 +36,7 @@ try { services.AddSingleton(AnsiConsole.Console); services.AddSingleton(); + services.AddSingleton(); }) .BuildApp() .RunAsync(args); @@ -51,206 +53,206 @@ finally await Log.CloseAndFlushAsync(); } -using var mp3Stream = new MemoryStream(); -using var mp4Stream = new FileStream(args[0], FileMode.Open, FileAccess.Read); +// using var mp3Stream = new MemoryStream(); +// using var mp4Stream = new FileStream(args[0], FileMode.Open, FileAccess.Read); -var wasExtracted = await FFMpegArguments - .FromPipeInput(new StreamPipeSource(mp4Stream)) - .OutputToPipe( - new StreamPipeSink(mp3Stream), - o => o.DisableChannel(Channel.Video).ForceFormat("mp3") - ) - .ProcessAsynchronously(); +// var wasExtracted = await FFMpegArguments +// .FromPipeInput(new StreamPipeSource(mp4Stream)) +// .OutputToPipe( +// new StreamPipeSink(mp3Stream), +// o => o.DisableChannel(Channel.Video).ForceFormat("mp3") +// ) +// .ProcessAsynchronously(); -// Step 2: Convert MP3 stream to 16khz wave format -mp3Stream.Position = 0; -using var reader = new Mp3FileReader(mp3Stream); -var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels); -using var resampler = new MediaFoundationResampler(reader, outFormat); -using var waveStream = new MemoryStream(); -WaveFileWriter.WriteWavFileToStream(waveStream, resampler); +// // Step 2: Convert MP3 stream to 16khz wave format +// mp3Stream.Position = 0; +// using var reader = new Mp3FileReader(mp3Stream); +// var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels); +// using var resampler = new MediaFoundationResampler(reader, outFormat); +// using var waveStream = new MemoryStream(); +// WaveFileWriter.WriteWavFileToStream(waveStream, resampler); -// Step 3: Split the wave stream into 2 minute segments -waveStream.Position = 0; -var segmentDuration = TimeSpan.FromMinutes(2); -var segments = new List(); -using var waveReader = new WaveFileReader(waveStream); -var segmentCount = (int)Math.Ceiling(waveReader.TotalTime.TotalMilliseconds / segmentDuration.TotalMilliseconds); +// // Step 3: Split the wave stream into 2 minute segments +// waveStream.Position = 0; +// var segmentDuration = TimeSpan.FromMinutes(2); +// var segments = new List(); +// using var waveReader = new WaveFileReader(waveStream); +// var segmentCount = (int)Math.Ceiling(waveReader.TotalTime.TotalMilliseconds / segmentDuration.TotalMilliseconds); -Directory.CreateDirectory("segments"); +// Directory.CreateDirectory("segments"); -foreach (var i in Enumerable.Range(0, segmentCount)) -{ - waveStream.Position = 0; - using var segmentWaveReader = new WaveFileReader(waveStream); - var segment = segmentWaveReader.ToSampleProvider() - .Skip(i * segmentDuration) - .Take(segmentDuration); - var segmentProvider = segment.ToWaveProvider16(); - var segmentStream = new MemoryStream(); - WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider); - segmentStream.Position = 0; - segments.Add(segmentStream); -} +// foreach (var i in Enumerable.Range(0, segmentCount)) +// { +// waveStream.Position = 0; +// using var segmentWaveReader = new WaveFileReader(waveStream); +// var segment = segmentWaveReader.ToSampleProvider() +// .Skip(i * segmentDuration) +// .Take(segmentDuration); +// var segmentProvider = segment.ToWaveProvider16(); +// var segmentStream = new MemoryStream(); +// WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider); +// segmentStream.Position = 0; +// segments.Add(segmentStream); +// } -// Step 4: Transcribe each segment using Whisper -using var modelMemoryStream = new MemoryStream(); -var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn); -await model.CopyToAsync(modelMemoryStream); -var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray()); -using var whisperProcessor = whisperFactory.CreateBuilder() - .WithLanguage("en") - .Build(); +// // Step 4: Transcribe each segment using Whisper +// using var modelMemoryStream = new MemoryStream(); +// var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn); +// await model.CopyToAsync(modelMemoryStream); +// var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray()); +// using var whisperProcessor = whisperFactory.CreateBuilder() +// .WithLanguage("en") +// .Build(); -var completeTranscription = new StringBuilder(); +// var completeTranscription = new StringBuilder(); -foreach (var (i, segment) in segments.Select((s, index) => (index, s))) -{ - var durationOffset = TimeSpan.FromMilliseconds(i * segmentDuration.TotalMilliseconds); - var segmentTranscription = new StringBuilder(); +// foreach (var (i, segment) in segments.Select((s, index) => (index, s))) +// { +// var durationOffset = TimeSpan.FromMilliseconds(i * segmentDuration.TotalMilliseconds); +// var segmentTranscription = new StringBuilder(); - await foreach (var result in whisperProcessor.ProcessAsync(segment, CancellationToken.None)) - { - var startTime = result.Start + durationOffset; - var endTime = result.End + durationOffset; - segmentTranscription.AppendLine(CultureInfo.CurrentCulture, $"[{startTime:hh\\:mm\\:ss} - {endTime:hh\\:mm\\:ss}] {result.Text}"); - } +// await foreach (var result in whisperProcessor.ProcessAsync(segment, CancellationToken.None)) +// { +// var startTime = result.Start + durationOffset; +// var endTime = result.End + durationOffset; +// segmentTranscription.AppendLine(CultureInfo.CurrentCulture, $"[{startTime:hh\\:mm\\:ss} - {endTime:hh\\:mm\\:ss}] {result.Text}"); +// } - completeTranscription.Append(segmentTranscription); -} +// completeTranscription.Append(segmentTranscription); +// } -// Step 5: Send the transcription to LLM for analysis -// TODO: Explore this prompt further...seems break -// when transcription is long -var prompt = $$""" -I need your help to transform my YouTube live stream transcript into engaging YouTube Shorts. Act as my content editor and pinpoint **all potential candidate segments** that are perfect for short-form video. I'm looking for clips that are: +// // Step 5: Send the transcription to LLM for analysis +// // TODO: Explore this prompt further...seems break +// // when transcription is long +// var prompt = $$""" +// I need your help to transform my YouTube live stream transcript into engaging YouTube Shorts. Act as my content editor and pinpoint **all potential candidate segments** that are perfect for short-form video. I'm looking for clips that are: -  * **Funny:** Moments that will make viewers laugh. -  * **Informative:** Sections packed with valuable information or tips. -  * **Insightful:** Portions offering unique perspectives or 'aha\!' moments. +//   * **Funny:** Moments that will make viewers laugh. +//   * **Informative:** Sections packed with valuable information or tips. +//   * **Insightful:** Portions offering unique perspectives or 'aha\!' moments. -For each suggested short, please provide: +// For each suggested short, please provide: -  * The **start time** of the initial segment and the **end time** of the final segment. The duration of each short should be no longer than 3 minutes, but **aim for durations between 15 seconds and 60 seconds**. However, the short **must be as long as necessary to capture the complete thought or idea**, even if it means exceeding the target range or extending slightly to capture all necessary dialogue. -  * A concise **title** that grabs attention. -  * A brief **description** highlighting the short's content and its appeal. -  * An **explanation** of why this particular segment is suitable for a YouTube Short, focusing on its potential for discoverability and engagement. +//   * The **start time** of the initial segment and the **end time** of the final segment. The duration of each short should be no longer than 3 minutes, but **aim for durations between 15 seconds and 60 seconds**. However, the short **must be as long as necessary to capture the complete thought or idea**, even if it means exceeding the target range or extending slightly to capture all necessary dialogue. +//   * A concise **title** that grabs attention. +//   * A brief **description** highlighting the short's content and its appeal. +//   * An **explanation** of why this particular segment is suitable for a YouTube Short, focusing on its potential for discoverability and engagement. -Please format your response as a JSON array of objects with the following structure: +// Please format your response as a JSON array of objects with the following structure: -```json -{ -  ""title"": ""string"", -  "start_time": "string", -  "end_time": "string", -  "description": "string", -  "explanation": "string" -} -``` +// ```json +// { +//   ""title"": ""string"", +//   "start_time": "string", +//   "end_time": "string", +//   "description": "string", +//   "explanation": "string" +// } +// ``` -Here is the transcript of my YouTube live stream: +// Here is the transcript of my YouTube live stream: -{{completeTranscription}} -"""; +// {{completeTranscription}} +// """; -if (string.IsNullOrWhiteSpace(apiKey)) -{ - throw new InvalidOperationException("GeminiApiKey is not configured in appsettings.json."); -} +// if (string.IsNullOrWhiteSpace(apiKey)) +// { +// throw new InvalidOperationException("GeminiApiKey is not configured in appsettings.json."); +// } -using var client = new HttpClient() -{ - Timeout = TimeSpan.FromMinutes(30) -}; +// using var client = new HttpClient() +// { +// Timeout = TimeSpan.FromMinutes(30) +// }; -var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={apiKey}"; -using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl) -{ - Content = new StringContent( - JsonSerializer.Serialize(new - { - contents = new[] - { - new - { - role = "user", - parts = new[] - { - new - { - text = prompt - } - }, - } - }, - generationConfig = new - { - responseMimeType = "application/json", - } - }), - Encoding.UTF8, - "application/json" - ) -}; -var response = await client.SendAsync(request); -var responseContent = await response.Content.ReadAsStringAsync(); -var responseJson = JsonSerializer.Deserialize(responseContent); -var candidatesText = responseJson? - .Candidates? - .FirstOrDefault()? - .Content - .Parts?.FirstOrDefault()? - .Text; -var analysis = JsonSerializer.Deserialize>(candidatesText ?? string.Empty); +// var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={apiKey}"; +// using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl) +// { +// Content = new StringContent( +// JsonSerializer.Serialize(new +// { +// contents = new[] +// { +// new +// { +// role = "user", +// parts = new[] +// { +// new +// { +// text = prompt +// } +// }, +// } +// }, +// generationConfig = new +// { +// responseMimeType = "application/json", +// } +// }), +// Encoding.UTF8, +// "application/json" +// ) +// }; +// var response = await client.SendAsync(request); +// var responseContent = await response.Content.ReadAsStringAsync(); +// var responseJson = JsonSerializer.Deserialize(responseContent); +// var candidatesText = responseJson? +// .Candidates? +// .FirstOrDefault()? +// .Content +// .Parts?.FirstOrDefault()? +// .Text; +// var analysis = JsonSerializer.Deserialize>(candidatesText ?? string.Empty); -if (analysis is null) -{ - Console.WriteLine(resourceManager.GetString("LLMAnalysisFailed", CultureInfo.CurrentCulture)); - return; -} +// if (analysis is null) +// { +// Console.WriteLine(resourceManager.GetString("LLMAnalysisFailed", CultureInfo.CurrentCulture)); +// return; +// } -foreach (var result in analysis) -{ - var fileName = string.Concat(result.Title.Split(Path.GetInvalidFileNameChars())); - await FFMpeg.SubVideoAsync( - args[0], - $"{fileName}.mp4", - result.StartTime, - result.EndTime - ); -} +// foreach (var result in analysis) +// { +// var fileName = string.Concat(result.Title.Split(Path.GetInvalidFileNameChars())); +// await FFMpeg.SubVideoAsync( +// args[0], +// $"{fileName}.mp4", +// result.StartTime, +// result.EndTime +// ); +// } -// Step 6: Use analysis to generate a short video +// // Step 6: Use analysis to generate a short video -record LLMAnalysis( - [property: JsonPropertyName("title")] - string Title, - [property: JsonPropertyName("description")] - string Description, - [property: JsonPropertyName("explanation")] - string Explanation, - [property: JsonPropertyName("start_time")] - TimeSpan StartTime, - [property: JsonPropertyName("end_time")] - TimeSpan EndTime -); +// record LLMAnalysis( +// [property: JsonPropertyName("title")] +// string Title, +// [property: JsonPropertyName("description")] +// string Description, +// [property: JsonPropertyName("explanation")] +// string Explanation, +// [property: JsonPropertyName("start_time")] +// TimeSpan StartTime, +// [property: JsonPropertyName("end_time")] +// TimeSpan EndTime +// ); -record LLMResponse( - [property: JsonPropertyName("candidates")] - Candidate[] Candidates -); +// record LLMResponse( +// [property: JsonPropertyName("candidates")] +// Candidate[] Candidates +// ); -record Candidate( - [property: JsonPropertyName("content")] - Content Content -); +// record Candidate( +// [property: JsonPropertyName("content")] +// Content Content +// ); -record Content( - [property: JsonPropertyName("parts")] - Part[] Parts -); +// record Content( +// [property: JsonPropertyName("parts")] +// Part[] Parts +// ); -record Part( - [property: JsonPropertyName("text")] - string Text -); \ No newline at end of file +// record Part( +// [property: JsonPropertyName("text")] +// string Text +// ); \ No newline at end of file diff --git a/src/StreamShorts.Console/StreamShorts.Console.csproj b/src/StreamShorts.Console/StreamShorts.Console.csproj index bd4e7f4..bfc9e32 100644 --- a/src/StreamShorts.Console/StreamShorts.Console.csproj +++ b/src/StreamShorts.Console/StreamShorts.Console.csproj @@ -25,4 +25,8 @@ PreserveNewest + + + + diff --git a/src/StreamShorts.Console/Usings.cs b/src/StreamShorts.Console/Usings.cs index 7f594c9..99671c7 100644 --- a/src/StreamShorts.Console/Usings.cs +++ b/src/StreamShorts.Console/Usings.cs @@ -2,7 +2,6 @@ global using System.ComponentModel; global using System.IO.Abstractions; global using System.Reflection; -global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; global using Microsoft.Extensions.Logging; @@ -15,4 +14,5 @@ global using Spectre.Console; global using Spectre.Console.Cli; global using StreamShorts.Console.Commands; -global using StreamShorts.Console.Hosting; \ No newline at end of file +global using StreamShorts.Console.Hosting; +global using StreamShorts.Library.Media.Audio; diff --git a/src/StreamShorts.Library/Media/Audio/AudioExtractor.cs b/src/StreamShorts.Library/Media/Audio/AudioExtractor.cs new file mode 100644 index 0000000..c43563a --- /dev/null +++ b/src/StreamShorts.Library/Media/Audio/AudioExtractor.cs @@ -0,0 +1,63 @@ +using FFMpegCore; +using FFMpegCore.Pipes; + +using Channel = FFMpegCore.Enums.Channel; + +namespace StreamShorts.Library.Media.Audio; + +/// +public class AudioExtractor : IAudioExtractor +{ + public async Task ExtractMp3FromMp4Async(Stream video) + { + if (video is null) + { + throw new ArgumentNullException(nameof(video), "Video stream cannot be null"); + } + + if (video.CanRead is false) + { + throw new ArgumentException("Video stream must be readable", nameof(video)); + } + + if (video.CanSeek is false) + { + throw new ArgumentException("Video stream must be seekable", nameof(video)); + } + + var originalPosition = video.Position; + + try + { + using var mp3Stream = new MemoryStream(); + using var mp4Stream = new MemoryStream(); + + await video.CopyToAsync(mp4Stream).ConfigureAwait(false); + mp4Stream.Position = 0; + + var wasExtracted = await FFMpegArguments + .FromPipeInput(new StreamPipeSource(mp4Stream)) + .OutputToPipe( + new StreamPipeSink(mp3Stream), + o => o.DisableChannel(Channel.Video).ForceFormat("mp3") + ) + .ProcessAsynchronously() + .ConfigureAwait(false); + + if (wasExtracted is false) + { + throw new FailedAudioExtractionException("Failed to extract audio from the video stream."); + } + + return mp3Stream; + } + catch (Exception e) when (e is not FailedAudioExtractionException) + { + throw new FailedAudioExtractionException("Failed to extract audio", e); + } + finally + { + video.Position = originalPosition; + } + } +} \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Audio/FailedAudioExtractionException.cs b/src/StreamShorts.Library/Media/Audio/FailedAudioExtractionException.cs new file mode 100644 index 0000000..25e57ec --- /dev/null +++ b/src/StreamShorts.Library/Media/Audio/FailedAudioExtractionException.cs @@ -0,0 +1,16 @@ +namespace StreamShorts.Library.Media.Audio; + +public class FailedAudioExtractionException : Exception +{ + public FailedAudioExtractionException() : base() + { + } + + public FailedAudioExtractionException(string message) : base(message) + { + } + + public FailedAudioExtractionException(string message, Exception innerException) : base(message, innerException) + { + } +} \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Audio/IAudioExtractor.cs b/src/StreamShorts.Library/Media/Audio/IAudioExtractor.cs index e69de29..9996292 100644 --- a/src/StreamShorts.Library/Media/Audio/IAudioExtractor.cs +++ b/src/StreamShorts.Library/Media/Audio/IAudioExtractor.cs @@ -0,0 +1,19 @@ +namespace StreamShorts.Library.Media.Audio; + +/// +/// Represents an interface for extracting audio from video streams. +/// +public interface IAudioExtractor +{ + /// + /// Extracts MP3 audio from an MP4 video stream. + /// + /// The input video stream. + /// A stream containing the extracted MP3 audio. + /// Thrown when the video stream is null. + /// Thrown when the video stream is not readable. + /// Thrown when the video stream is not seekable. + /// Thrown when the audio extraction fails. + /// The method will preserve the passed video stream's data and position. + Task ExtractMp3FromMp4Async(Stream video); +} \ No newline at end of file diff --git a/src/StreamShorts.Library/StreamShorts.Library.csproj b/src/StreamShorts.Library/StreamShorts.Library.csproj index c632161..7c67d16 100644 --- a/src/StreamShorts.Library/StreamShorts.Library.csproj +++ b/src/StreamShorts.Library/StreamShorts.Library.csproj @@ -1,3 +1,5 @@  - + + + diff --git a/tests/.editorconfig b/tests/.editorconfig index 79bfd7f..b7ce383 100644 --- a/tests/.editorconfig +++ b/tests/.editorconfig @@ -1,2 +1,3 @@ [*.cs] dotnet_diagnostic.CA1707.severity = none +dotnet_diagnostic.CA2007.severity = none diff --git a/tests/StreamShorts.Library.Tests/StreamShorts.Library.Tests.csproj b/tests/StreamShorts.Library.Tests/StreamShorts.Library.Tests.csproj index 0dfcb36..68ccb7f 100644 --- a/tests/StreamShorts.Library.Tests/StreamShorts.Library.Tests.csproj +++ b/tests/StreamShorts.Library.Tests/StreamShorts.Library.Tests.csproj @@ -1,7 +1,9 @@ + + diff --git a/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioExtractorTests.cs b/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioExtractorTests.cs new file mode 100644 index 0000000..7736e2b --- /dev/null +++ b/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioExtractorTests.cs @@ -0,0 +1,36 @@ +namespace StreamShorts.Library.Tests.Unit.Media.Audio; + +public class AudioExtractorTests +{ + private readonly AudioExtractor _sut = new(); + + [Fact] + public async Task ExtractMp3FromMp4Async_WhenVideoIsNull_ItShouldThrow() + { + var action = async () => await _sut.ExtractMp3FromMp4Async(null!); + + await action.Should().ThrowAsync(); + } + + [Fact] + public async Task ExtractMp3FromMp4Async_WhenVideoIsNotReadable_ItShouldThrow() + { + var mockStream = new Mock(); + mockStream.Setup(s => s.CanRead).Returns(false); + + var action = async () => await _sut.ExtractMp3FromMp4Async(mockStream.Object); + + await action.Should().ThrowAsync(); + } + + [Fact] + public async Task ExtractMp3FromMp4Async_WhenVideoIsNotSeekable_ItShouldThrow() + { + var mockStream = new Mock(); + mockStream.Setup(s => s.CanSeek).Returns(false); + + var action = async () => await _sut.ExtractMp3FromMp4Async(mockStream.Object); + + await action.Should().ThrowAsync(); + } +} \ No newline at end of file diff --git a/tests/StreamShorts.Library.Tests/Usings.cs b/tests/StreamShorts.Library.Tests/Usings.cs new file mode 100644 index 0000000..0900feb --- /dev/null +++ b/tests/StreamShorts.Library.Tests/Usings.cs @@ -0,0 +1,5 @@ +global using AwesomeAssertions; + +global using Moq; + +global using StreamShorts.Library.Media.Audio;