diff --git a/.vscode/settings.json b/.vscode/settings.json index 30189f9..fdd0136 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "cSpell.words": [ + "Ggml", "resampler", "resx" ] diff --git a/src/StreamShorts.Console/Commands/DefaultCommand.cs b/src/StreamShorts.Console/Commands/DefaultCommand.cs index 9fe3b9d..6fe0ea1 100644 --- a/src/StreamShorts.Console/Commands/DefaultCommand.cs +++ b/src/StreamShorts.Console/Commands/DefaultCommand.cs @@ -4,13 +4,13 @@ internal class DefaultCommand( IFileSystem fileSystem, IAnsiConsole console, IAudioExtractor audioExtractor, - IAudioConverter audioConverter + ITranscriber transcriber ) : AsyncCommand { private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); private readonly IAnsiConsole _console = console ?? throw new ArgumentNullException(nameof(console)); private readonly IAudioExtractor _audioExtractor = audioExtractor ?? throw new ArgumentNullException(nameof(audioExtractor)); - private readonly IAudioConverter _audioConverter = audioConverter ?? throw new ArgumentNullException(nameof(audioConverter)); + private readonly ITranscriber _transcriber = transcriber ?? throw new ArgumentNullException(nameof(transcriber)); internal class Settings : CommandSettings { @@ -63,22 +63,21 @@ internal class DefaultCommand( _console.MarkupLine($"[blue]Audio extracted[/] [green]successfully![/]"); - Stream? wavStream = null; + List transcriptionSegments = []; - _console.Status() + await _console.Status() .Spinner(Spinner.Known.Dots) - .Start("Converting audio...", ctx => + .StartAsync("Transcribing audio...", async ctx => { - wavStream = _audioConverter.ConvertMp3ToWav16(audioStream); + await foreach (var segment in _transcriber.TranscribeAsync(audioStream)) + { + transcriptionSegments.Add(segment); + var segmentTimeText = $"[{segment.StartTime:hh\\:mm\\:ss} - {segment.EndTime:hh\\:mm\\:ss}]"; + ctx.Status($"Transcribed segment {segmentTimeText.EscapeMarkup()}"); + } }); - if (wavStream is null) - { - _console.MarkupLine("[red]Failed[/] to convert audio to WAV format."); - return 2; - } - - _console.MarkupLine($"[blue]Audio converted[/] [green]successfully![/]"); + _console.MarkupLine($"[blue]Transcription completed[/] [green]successfully![/]"); return 0; } diff --git a/src/StreamShorts.Console/Program.cs b/src/StreamShorts.Console/Program.cs index 1743853..a8b4730 100644 --- a/src/StreamShorts.Console/Program.cs +++ b/src/StreamShorts.Console/Program.cs @@ -1,20 +1,4 @@ -using System.Globalization; -using System.Resources; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; - -using FFMpegCore; -using FFMpegCore.Enums; -using FFMpegCore.Pipes; - -using NAudio.Wave; - - -using Whisper.net; -using Whisper.net.Ggml; - -Log.Logger = new LoggerConfiguration() +Log.Logger = new LoggerConfiguration() .WriteTo.File( formatter: new CompactJsonFormatter(), path: Path.Combine(AppContext.BaseDirectory, "logs", "log.jsonl"), @@ -37,7 +21,7 @@ try services.AddSingleton(AnsiConsole.Console); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); }) .BuildApp() .RunAsync(args); diff --git a/src/StreamShorts.Console/StreamShorts.Console.csproj b/src/StreamShorts.Console/StreamShorts.Console.csproj index bfc9e32..ecca060 100644 --- a/src/StreamShorts.Console/StreamShorts.Console.csproj +++ b/src/StreamShorts.Console/StreamShorts.Console.csproj @@ -5,11 +5,9 @@ - - diff --git a/src/StreamShorts.Console/Usings.cs b/src/StreamShorts.Console/Usings.cs index 99671c7..0f992ee 100644 --- a/src/StreamShorts.Console/Usings.cs +++ b/src/StreamShorts.Console/Usings.cs @@ -16,3 +16,4 @@ global using Spectre.Console.Cli; global using StreamShorts.Console.Commands; global using StreamShorts.Console.Hosting; global using StreamShorts.Library.Media.Audio; +global using StreamShorts.Library.Transcription; \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Audio/AudioConverter.cs b/src/StreamShorts.Library/Media/Audio/AudioConverter.cs deleted file mode 100644 index b90245e..0000000 --- a/src/StreamShorts.Library/Media/Audio/AudioConverter.cs +++ /dev/null @@ -1,18 +0,0 @@ - -using NAudio.Wave; - -namespace StreamShorts.Library.Media.Audio; - -public class AudioConverter : IAudioConverter -{ - public Stream ConvertMp3ToWav16(Stream mp3) - { - using var reader = new Mp3FileReader(mp3); - var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels); - using var resampler = new MediaFoundationResampler(reader, outFormat); - var waveStream = new MemoryStream(); - WaveFileWriter.WriteWavFileToStream(waveStream, resampler); - waveStream.Position = 0; - return waveStream; - } -} \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/Audio/AudioExtractor.cs b/src/StreamShorts.Library/Media/Audio/AudioExtractor.cs index a78b13d..19c1f07 100644 --- a/src/StreamShorts.Library/Media/Audio/AudioExtractor.cs +++ b/src/StreamShorts.Library/Media/Audio/AudioExtractor.cs @@ -1,17 +1,28 @@ namespace StreamShorts.Library.Media.Audio; +/// +/// Extracts audio from video files. +/// /// public class AudioExtractor : IAudioExtractor { - private readonly IFFMpegService _ffmpegService = new FFMpegService(); + private readonly IVideoService _videoService = new FFMpegService(); + /// + /// Initializes a new instance of the class. + /// public AudioExtractor() { } - internal AudioExtractor(IFFMpegService ffmpegService) + /// + /// Initializes a new instance of the class with a specified FFMpeg service. + /// + /// The video service to use for audio extraction. + /// Thrown when the FFMpeg service is null. + internal AudioExtractor(IVideoService videoService) { - _ffmpegService = ffmpegService; + _videoService = videoService ?? throw new ArgumentNullException(nameof(videoService), $"{nameof(videoService)} cannot be null"); } public async Task ExtractMp3FromMp4Async(Stream video) @@ -41,7 +52,7 @@ public class AudioExtractor : IAudioExtractor await video.CopyToAsync(mp4Stream).ConfigureAwait(false); mp4Stream.Position = 0; - var wasExtracted = await _ffmpegService.ExtractAudioFromVideoAsync(mp4Stream, mp3Stream).ConfigureAwait(false); + var wasExtracted = await _videoService.ExtractAudioFromVideoAsync(mp4Stream, mp3Stream).ConfigureAwait(false); if (wasExtracted is false) { diff --git a/src/StreamShorts.Library/Media/Audio/IAudioConverter.cs b/src/StreamShorts.Library/Media/Audio/IAudioConverter.cs deleted file mode 100644 index 2022b31..0000000 --- a/src/StreamShorts.Library/Media/Audio/IAudioConverter.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StreamShorts.Library.Media.Audio; - -public interface IAudioConverter -{ - Stream ConvertMp3ToWav16(Stream mp3); -} \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/FFMpegService.cs b/src/StreamShorts.Library/Media/FFMpegService.cs index 509de62..5c12006 100644 --- a/src/StreamShorts.Library/Media/FFMpegService.cs +++ b/src/StreamShorts.Library/Media/FFMpegService.cs @@ -6,8 +6,11 @@ using FFMpegCore.Pipes; namespace StreamShorts.Library.Media; -[ExcludeFromCodeCoverage] -internal class FFMpegService : IFFMpegService +/// +/// Represents a service for processing video files using FFMpeg. +/// +/// +internal class FFMpegService : IVideoService { public async Task ExtractAudioFromVideoAsync(Stream video, Stream audio) { diff --git a/src/StreamShorts.Library/Media/IAudioService.cs b/src/StreamShorts.Library/Media/IAudioService.cs new file mode 100644 index 0000000..fee0713 --- /dev/null +++ b/src/StreamShorts.Library/Media/IAudioService.cs @@ -0,0 +1,32 @@ + +namespace StreamShorts.Library.Media; + +/// +/// Represents a service for processing audio files. +/// +internal interface IAudioService +{ + /// + /// Converts an MP3 stream to a WAV stream with a 16 kHz sample rate. + /// + /// The input MP3 stream. + /// The output WAV stream. + Stream ConvertMp3ToWav16(Stream mp3); + + /// + /// Gets the number of segments in a WAV stream based on the specified segment duration. + /// + /// param name="wavStream">The input WAV stream. + /// The duration of each segment. + /// The number of segments. + int GetNumberOfWavSegments(Stream wavStream, TimeSpan segmentDuration); + + /// + /// Gets a segment of a WAV stream based on the specified segment number and duration. + /// + /// param name="wavStream">The input WAV stream. + /// The segment number to retrieve. + /// The duration of each segment. + /// The segment stream. + Stream GetWavSegment(Stream wavStream, int segmentNumber, TimeSpan segmentDuration); +} \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/IFFMpegService.cs b/src/StreamShorts.Library/Media/IFFMpegService.cs deleted file mode 100644 index 50e10c8..0000000 --- a/src/StreamShorts.Library/Media/IFFMpegService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StreamShorts.Library.Media; - -internal interface IFFMpegService -{ - Task ExtractAudioFromVideoAsync(Stream video, Stream audio); -} \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/IVideoService.cs b/src/StreamShorts.Library/Media/IVideoService.cs new file mode 100644 index 0000000..70c3d1b --- /dev/null +++ b/src/StreamShorts.Library/Media/IVideoService.cs @@ -0,0 +1,15 @@ +namespace StreamShorts.Library.Media; + +/// +/// Represents a service for processing video files. +/// +internal interface IVideoService +{ + /// + /// Extracts audio from a video stream and writes it to an audio stream. + /// + /// The input video stream. + /// The output audio stream. + /// A task that represents the asynchronous operation. The task result indicates whether the extraction was successful. + Task ExtractAudioFromVideoAsync(Stream video, Stream audio); +} \ No newline at end of file diff --git a/src/StreamShorts.Library/Media/NAudioService.cs b/src/StreamShorts.Library/Media/NAudioService.cs new file mode 100644 index 0000000..e715c5e --- /dev/null +++ b/src/StreamShorts.Library/Media/NAudioService.cs @@ -0,0 +1,45 @@ + +using NAudio.Wave; + +namespace StreamShorts.Library.Media; + +/// +/// Represents a service for processing audio files using NAudio. +/// +/// +internal class NAudioService : IAudioService +{ + public Stream ConvertMp3ToWav16(Stream mp3) + { + using var reader = new Mp3FileReader(mp3); + var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels); + using var resampler = new MediaFoundationResampler(reader, outFormat); + var waveStream = new MemoryStream(); + WaveFileWriter.WriteWavFileToStream(waveStream, resampler); + waveStream.Position = 0; + return waveStream; + } + + public int GetNumberOfWavSegments(Stream wavStream, TimeSpan segmentDuration) + { + using var waveReader = new WaveFileReader(wavStream); + var totalDuration = waveReader.TotalTime; + var segmentCount = (int)Math.Ceiling(totalDuration.TotalMilliseconds / segmentDuration.TotalMilliseconds); + wavStream.Position = 0; + return segmentCount; + } + + public Stream GetWavSegment(Stream wavStream, int segmentNumber, TimeSpan segmentDuration) + { + using var segmentWaveReader = new WaveFileReader(wavStream); + var segment = segmentWaveReader.ToSampleProvider() + .Skip(segmentNumber * segmentDuration) + .Take(segmentDuration); + var segmentProvider = segment.ToWaveProvider16(); + var segmentStream = new MemoryStream(); + WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider); + segmentStream.Position = 0; + wavStream.Position = 0; + return segmentStream; + } +} \ No newline at end of file diff --git a/src/StreamShorts.Library/StreamShorts.Library.csproj b/src/StreamShorts.Library/StreamShorts.Library.csproj index 281f941..97627db 100644 --- a/src/StreamShorts.Library/StreamShorts.Library.csproj +++ b/src/StreamShorts.Library/StreamShorts.Library.csproj @@ -2,5 +2,6 @@ + diff --git a/src/StreamShorts.Library/Transcription/ITranscriber.cs b/src/StreamShorts.Library/Transcription/ITranscriber.cs index 425da8d..39572bd 100644 --- a/src/StreamShorts.Library/Transcription/ITranscriber.cs +++ b/src/StreamShorts.Library/Transcription/ITranscriber.cs @@ -1,12 +1,17 @@ +using System.Runtime.CompilerServices; + namespace StreamShorts.Library.Transcription; +/// +/// Represents a transcriber interface for audio transcription. +/// public interface ITranscriber { - IAsyncEnumerable TranscribeAsync(Stream audio); + /// + /// Transcribes the audio stream into text segments. + /// + /// The audio stream to transcribe. + /// A cancellation token to cancel the operation. + /// An where T is . + IAsyncEnumerable TranscribeAsync(Stream audio, CancellationToken cancellationToken = default); } - -public record TranscriptionSegment( - TimeSpan StartTime, - TimeSpan EndTime, - string Text -); \ No newline at end of file diff --git a/src/StreamShorts.Library/Transcription/TranscriptionSegment.cs b/src/StreamShorts.Library/Transcription/TranscriptionSegment.cs new file mode 100644 index 0000000..bd5a232 --- /dev/null +++ b/src/StreamShorts.Library/Transcription/TranscriptionSegment.cs @@ -0,0 +1,13 @@ +namespace StreamShorts.Library.Transcription; + +/// +/// Represents a segment of transcribed audio. +/// +/// The start time of the segment. +/// The end time of the segment. +/// The transcribed text of the segment. +public record TranscriptionSegment( + TimeSpan StartTime, + TimeSpan EndTime, + string Text +); \ No newline at end of file diff --git a/src/StreamShorts.Library/Transcription/WhisperTranscriber.cs b/src/StreamShorts.Library/Transcription/WhisperTranscriber.cs index c77a20a..f264df5 100644 --- a/src/StreamShorts.Library/Transcription/WhisperTranscriber.cs +++ b/src/StreamShorts.Library/Transcription/WhisperTranscriber.cs @@ -1,10 +1,79 @@ +using System.Runtime.CompilerServices; + +using StreamShorts.Library.Media; + +using Whisper.net; +using Whisper.net.Ggml; namespace StreamShorts.Library.Transcription; +/// +/// Represents a transcriber that uses Whisper for audio transcription. +/// +/// public class WhisperTranscriber : ITranscriber { - public IAsyncEnumerable TranscribeAsync(Stream audio) + private readonly IAudioService _audioService = new NAudioService(); + private WhisperProcessor? _whisperProcessor; + + /// + /// Initializes a new instance of the class. + /// + public WhisperTranscriber() { - throw new NotImplementedException(); + } + + /// + /// Initializes a new instance of the class with a specified audio service. + /// + /// The audio service to use for audio processing. + /// Thrown when the audio service is null. TranscribeAsync(Stream audio, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var segmentDuration = TimeSpan.FromMinutes(2); + var wavStream = _audioService.ConvertMp3ToWav16(audio); + var numberOfSegments = _audioService.GetNumberOfWavSegments(wavStream, segmentDuration); + + foreach (var segmentNumber in Enumerable.Range(0, numberOfSegments)) + { + var segmentStream = _audioService.GetWavSegment(wavStream, segmentNumber, segmentDuration); + var durationOffset = TimeSpan.FromMilliseconds(segmentNumber * segmentDuration.TotalMilliseconds); + + await foreach (var result in ProcessSegmentAsync(segmentStream, cancellationToken).ConfigureAwait(false)) + { + yield return new TranscriptionSegment( + result.Start + durationOffset, + result.End + durationOffset, + result.Text + ); + } + } + } + + private async IAsyncEnumerable ProcessSegmentAsync( + Stream segmentStream, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (_whisperProcessor is null) + { + + using var modelMemoryStream = new MemoryStream(); + var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn, cancellationToken: cancellationToken).ConfigureAwait(false); + await model.CopyToAsync(modelMemoryStream, cancellationToken).ConfigureAwait(false); + var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray()); + _whisperProcessor = whisperFactory.CreateBuilder() + .WithLanguage("en") + .Build(); + } + + await foreach (var result in _whisperProcessor.ProcessAsync(segmentStream, cancellationToken).ConfigureAwait(false)) + { + yield return result; + } } } \ No newline at end of file diff --git a/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioConverterTests.cs b/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioConverterTests.cs deleted file mode 100644 index 886d185..0000000 --- a/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioConverterTests.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace StreamShorts.Library.Tests.Unit.Media.Audio; - -public class AudioConverterTests -{ - [Fact] - public void Test() - { - return; - } -} \ No newline at end of file diff --git a/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioExtractorTests.cs b/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioExtractorTests.cs index d629a2f..d70f76b 100644 --- a/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioExtractorTests.cs +++ b/tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioExtractorTests.cs @@ -6,7 +6,7 @@ namespace StreamShorts.Library.Tests.Unit.Media.Audio; public class AudioExtractorTests { - private readonly Mock _mockFfmpegService = new(); + private readonly Mock _mockFfmpegService = new(); private readonly AudioExtractor _sut; public AudioExtractorTests() @@ -14,6 +14,14 @@ public class AudioExtractorTests _sut = new(_mockFfmpegService.Object); } + [Fact] + public void Constructor_WhenCalledWithNullFfmpegService_ItShouldThrow() + { + var action = () => new AudioExtractor(null!); + + action.Should().Throw(); + } + [Fact] public async Task ExtractMp3FromMp4Async_WhenVideoIsNull_ItShouldThrow() {