diff --git a/src/Directory.packages.props b/src/Directory.packages.props index d727d6b..46ac85d 100644 --- a/src/Directory.packages.props +++ b/src/Directory.packages.props @@ -1,7 +1,10 @@ - true - - + + + + + + \ No newline at end of file diff --git a/src/StreamShorts.Console/.editorconfig b/src/StreamShorts.Console/.editorconfig new file mode 100644 index 0000000..79a8d3d --- /dev/null +++ b/src/StreamShorts.Console/.editorconfig @@ -0,0 +1,2 @@ +[*.cs] +dotnet_diagnostic.CA2007.severity = none \ No newline at end of file diff --git a/src/StreamShorts.Console/Program.cs b/src/StreamShorts.Console/Program.cs index 82d8e79..b98e09e 100644 --- a/src/StreamShorts.Console/Program.cs +++ b/src/StreamShorts.Console/Program.cs @@ -1,7 +1,99 @@ -// See https://aka.ms/new-console-template for more information -using System.Globalization; +using System.Globalization; using System.Resources; +using System.Text; + +using FFMpegCore; +using FFMpegCore.Enums; +using FFMpegCore.Pipes; + +using NAudio.Wave; + +using Whisper.net; +using Whisper.net.Ggml; var resourceManager = new ResourceManager("StreamShorts.Console.Resources.Resources", typeof(Program).Assembly); Console.WriteLine(resourceManager.GetString("WelcomeMessage", CultureInfo.CurrentCulture)); + +// Step 1: Retrieve stream +if (args.Length is 0) +{ + Console.WriteLine(resourceManager.GetString("StreamNotProvided", CultureInfo.CurrentCulture)); + return; +} + +using var mp3Stream = new MemoryStream(); +using var mp4Stream = new FileStream(args[0], FileMode.Open, FileAccess.Read); + +var wasExtracted = await FFMpegArguments + .FromPipeInput(new StreamPipeSource(mp4Stream)) + .OutputToPipe( + new StreamPipeSink(mp3Stream), + o => o.DisableChannel(Channel.Video).ForceFormat("mp3") + ) + .ProcessAsynchronously(); + +if (wasExtracted is false) +{ + Console.WriteLine(resourceManager.GetString("Mp3ExtractionFailed", CultureInfo.CurrentCulture)); + return; +} + +// Step 2: Convert MP3 stream to 16khz wave format +mp3Stream.Position = 0; +using var reader = new Mp3FileReader(mp3Stream); +var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels); +using var resampler = new MediaFoundationResampler(reader, outFormat); +using var waveStream = new MemoryStream(); +WaveFileWriter.WriteWavFileToStream(waveStream, resampler); + +// Step 3: Split the wave stream into 2 minute segments +waveStream.Position = 0; +var segmentDuration = TimeSpan.FromMinutes(2); +var segments = new List(); +using var waveReader = new WaveFileReader(waveStream); +var segmentCount = (int)Math.Ceiling(waveReader.TotalTime.TotalMilliseconds / segmentDuration.TotalMilliseconds); + +Directory.CreateDirectory("segments"); + +foreach (var i in Enumerable.Range(0, segmentCount)) +{ + waveStream.Position = 0; + using var segmentWaveReader = new WaveFileReader(waveStream); + var segment = segmentWaveReader.ToSampleProvider() + .Skip(i * segmentDuration) + .Take(segmentDuration); + var segmentProvider = segment.ToWaveProvider16(); + var segmentStream = new MemoryStream(); + WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider); + segmentStream.Position = 0; + segments.Add(segmentStream); +} + +// Step 4: Transcribe each segment using Whisper +using var modelMemoryStream = new MemoryStream(); +var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn); +await model.CopyToAsync(modelMemoryStream); +var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray()); +using var whisperProcessor = whisperFactory.CreateBuilder() + .WithLanguage("en") + .Build(); + +foreach (var (i, segment) in segments.Select((s, index) => (index, s))) +{ + var durationOffset = TimeSpan.FromMilliseconds(i * segmentDuration.TotalMilliseconds); + var segmentTranscription = new StringBuilder(); + + await foreach (var result in whisperProcessor.ProcessAsync(segment, CancellationToken.None)) + { + var startTime = result.Start + durationOffset; + var endTime = result.End + durationOffset; + segmentTranscription.AppendLine(CultureInfo.CurrentCulture, $"[{startTime:hh\\:mm\\:ss} - {endTime:hh\\:mm\\:ss}] {result.Text}"); + } + + await File.AppendAllTextAsync("transcription.txt", segmentTranscription.ToString()); +} + +// Step 5: Send the transcription to LLM for analysis + +// Step 6: Use analysis to generate a short video \ No newline at end of file diff --git a/src/StreamShorts.Console/Resources/Resources.resx b/src/StreamShorts.Console/Resources/Resources.resx index 5a85d0b..c866a39 100644 --- a/src/StreamShorts.Console/Resources/Resources.resx +++ b/src/StreamShorts.Console/Resources/Resources.resx @@ -3,4 +3,10 @@ Welcome to StreamShorts! + + No stream provided. Please specify a stream to process. + + + Failed to extract MP3 from the provided stream. + \ No newline at end of file diff --git a/src/StreamShorts.Console/StreamShorts.Console.csproj b/src/StreamShorts.Console/StreamShorts.Console.csproj index 6a70a1a..38b1fd9 100644 --- a/src/StreamShorts.Console/StreamShorts.Console.csproj +++ b/src/StreamShorts.Console/StreamShorts.Console.csproj @@ -4,4 +4,10 @@ Exe + + + + + +