feat: initial whipser transcriber implementation

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "cSpell.words": [
+    "Ggml",
     "resampler",
     "resx"
   ]

src/StreamShorts.Console/Commands/DefaultCommand.cs

@@ -4,13 +4,13 @@ internal class DefaultCommand(
   IFileSystem fileSystem,
   IAnsiConsole console,
   IAudioExtractor audioExtractor,
-  IAudioConverter audioConverter
+  ITranscriber transcriber
 ) : AsyncCommand<DefaultCommand.Settings>
 {
   private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
   private readonly IAnsiConsole _console = console ?? throw new ArgumentNullException(nameof(console));
   private readonly IAudioExtractor _audioExtractor = audioExtractor ?? throw new ArgumentNullException(nameof(audioExtractor));
-  private readonly IAudioConverter _audioConverter = audioConverter ?? throw new ArgumentNullException(nameof(audioConverter));
+  private readonly ITranscriber _transcriber = transcriber ?? throw new ArgumentNullException(nameof(transcriber));
 
   internal class Settings : CommandSettings
   {
@@ -63,22 +63,21 @@ internal class DefaultCommand(
 
     _console.MarkupLine($"[blue]Audio extracted[/] [green]successfully![/]");
 
-    Stream? wavStream = null;
+    List<TranscriptionSegment> transcriptionSegments = [];
 
-    _console.Status()
+    await _console.Status()
       .Spinner(Spinner.Known.Dots)
-      .Start("Converting audio...", ctx =>
+      .StartAsync("Transcribing audio...", async ctx =>
       {
-        wavStream = _audioConverter.ConvertMp3ToWav16(audioStream);
+        await foreach (var segment in _transcriber.TranscribeAsync(audioStream))
+        {
+          transcriptionSegments.Add(segment);
+          var segmentTimeText = $"[{segment.StartTime:hh\\:mm\\:ss} - {segment.EndTime:hh\\:mm\\:ss}]";
+          ctx.Status($"Transcribed segment {segmentTimeText.EscapeMarkup()}");
+        }
       });
 
-    if (wavStream is null)
+    _console.MarkupLine($"[blue]Transcription completed[/] [green]successfully![/]");
-    {
-      _console.MarkupLine("[red]Failed[/] to convert audio to WAV format.");
-      return 2;
-    }
-
-    _console.MarkupLine($"[blue]Audio converted[/] [green]successfully![/]");
 
     return 0;
   }

src/StreamShorts.Console/Program.cs

@@ -1,20 +1,4 @@
-using System.Globalization;
+Log.Logger = new LoggerConfiguration()
-using System.Resources;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-using FFMpegCore;
-using FFMpegCore.Enums;
-using FFMpegCore.Pipes;
-
-using NAudio.Wave;
-
-
-using Whisper.net;
-using Whisper.net.Ggml;
-
-Log.Logger = new LoggerConfiguration()
   .WriteTo.File(
     formatter: new CompactJsonFormatter(),
     path: Path.Combine(AppContext.BaseDirectory, "logs", "log.jsonl"),
@@ -37,7 +21,7 @@ try
       services.AddSingleton(AnsiConsole.Console);
       services.AddSingleton<IFileSystem, FileSystem>();
       services.AddSingleton<IAudioExtractor, AudioExtractor>();
-      services.AddSingleton<IAudioConverter, AudioConverter>();
+      services.AddSingleton<ITranscriber, WhisperTranscriber>();
     })
     .BuildApp()
     .RunAsync(args);

src/StreamShorts.Console/StreamShorts.Console.csproj

@@ -5,11 +5,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FFMpegCore" />
     <PackageReference Include="Microsoft.Extensions.Configuration" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" />
     <PackageReference Include="Microsoft.Extensions.Hosting" />
-    <PackageReference Include="NAudio" />
     <PackageReference Include="Serilog" />
     <PackageReference Include="Serilog.Extensions.Hosting" />
     <PackageReference Include="Serilog.Formatting.Compact" />

src/StreamShorts.Console/Usings.cs

@@ -16,3 +16,4 @@ global using Spectre.Console.Cli;
 global using StreamShorts.Console.Commands;
 global using StreamShorts.Console.Hosting;
 global using StreamShorts.Library.Media.Audio;
+global using StreamShorts.Library.Transcription;

src/StreamShorts.Library/Media/Audio/AudioConverter.cs

@@ -1,18 +0,0 @@
-
-using NAudio.Wave;
-
-namespace StreamShorts.Library.Media.Audio;
-
-public class AudioConverter : IAudioConverter
-{
-  public Stream ConvertMp3ToWav16(Stream mp3)
-  {
-    using var reader = new Mp3FileReader(mp3);
-    var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels);
-    using var resampler = new MediaFoundationResampler(reader, outFormat);
-    var waveStream = new MemoryStream();
-    WaveFileWriter.WriteWavFileToStream(waveStream, resampler);
-    waveStream.Position = 0;
-    return waveStream;
-  }
-}

src/StreamShorts.Library/Media/Audio/AudioExtractor.cs

@@ -1,17 +1,28 @@
 namespace StreamShorts.Library.Media.Audio;
 
+/// <summary>
+/// Extracts audio from video files.
+/// </summary>
 /// <inheritdoc/>
 public class AudioExtractor : IAudioExtractor
 {
-  private readonly IFFMpegService _ffmpegService = new FFMpegService();
+  private readonly IVideoService _videoService = new FFMpegService();
 
+  /// <summary>
+  /// Initializes a new instance of the <see cref="AudioExtractor"/> class.
+  /// </summary>
   public AudioExtractor()
   {
   }
 
-  internal AudioExtractor(IFFMpegService ffmpegService)
+  /// <summary>
+  /// Initializes a new instance of the <see cref="AudioExtractor"/> class with a specified FFMpeg service.
+  /// </summary>
+  /// <param name="videoService">The video service to use for audio extraction.</param>
+  /// <exception cref="ArgumentNullException">Thrown when the FFMpeg service is null.</exception>
+  internal AudioExtractor(IVideoService videoService)
   {
-    _ffmpegService = ffmpegService;
+    _videoService = videoService ?? throw new ArgumentNullException(nameof(videoService), $"{nameof(videoService)} cannot be null");
   }
 
   public async Task<Stream> ExtractMp3FromMp4Async(Stream video)
@@ -41,7 +52,7 @@ public class AudioExtractor : IAudioExtractor
       await video.CopyToAsync(mp4Stream).ConfigureAwait(false);
       mp4Stream.Position = 0;
 
-      var wasExtracted = await _ffmpegService.ExtractAudioFromVideoAsync(mp4Stream, mp3Stream).ConfigureAwait(false);
+      var wasExtracted = await _videoService.ExtractAudioFromVideoAsync(mp4Stream, mp3Stream).ConfigureAwait(false);
 
       if (wasExtracted is false)
       {

src/StreamShorts.Library/Media/Audio/IAudioConverter.cs

@@ -1,6 +0,0 @@
-namespace StreamShorts.Library.Media.Audio;
-
-public interface IAudioConverter
-{
-  Stream ConvertMp3ToWav16(Stream mp3);
-}

src/StreamShorts.Library/Media/FFMpegService.cs

@@ -6,8 +6,11 @@ using FFMpegCore.Pipes;
 
 namespace StreamShorts.Library.Media;
 
-[ExcludeFromCodeCoverage]
+/// <summary>
-internal class FFMpegService : IFFMpegService
+/// Represents a service for processing video files using FFMpeg.
+/// </summary>
+/// <inheritdoc/>
+internal class FFMpegService : IVideoService
 {
   public async Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio)
   {

src/StreamShorts.Library/Media/IAudioService.cs

@@ -0,0 +1,32 @@
+
+namespace StreamShorts.Library.Media;
+
+/// <summary>
+/// Represents a service for processing audio files.
+/// </summary>
+internal interface IAudioService
+{
+  /// <summary>
+  /// Converts an MP3 stream to a WAV stream with a 16 kHz sample rate.
+  /// </summary>
+  /// <param name="mp3">The input MP3 stream.</param>
+  /// <returns>The output WAV stream.</returns>
+  Stream ConvertMp3ToWav16(Stream mp3);
+
+  /// <summary>
+  /// Gets the number of segments in a WAV stream based on the specified segment duration.
+  /// </summary>
+  /// param name="wavStream">The input WAV stream.</param>
+  /// <param name="segmentDuration">The duration of each segment.</param>
+  /// <returns>The number of segments.</returns>
+  int GetNumberOfWavSegments(Stream wavStream, TimeSpan segmentDuration);
+
+  /// <summary>
+  /// Gets a segment of a WAV stream based on the specified segment number and duration.
+  /// </summary>
+  /// param name="wavStream">The input WAV stream.</param>
+  /// <param name="segmentNumber">The segment number to retrieve.</param>
+  /// <param name="segmentDuration">The duration of each segment.</param>
+  /// <returns>The segment stream.</returns>
+  Stream GetWavSegment(Stream wavStream, int segmentNumber, TimeSpan segmentDuration);
+}

src/StreamShorts.Library/Media/IFFMpegService.cs

@@ -1,6 +0,0 @@
-namespace StreamShorts.Library.Media;
-
-internal interface IFFMpegService
-{
-  Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio);
-}

src/StreamShorts.Library/Media/IVideoService.cs

@@ -0,0 +1,15 @@
+namespace StreamShorts.Library.Media;
+
+/// <summary>
+/// Represents a service for processing video files.
+/// </summary>
+internal interface IVideoService
+{
+  /// <summary>
+  /// Extracts audio from a video stream and writes it to an audio stream.
+  /// </summary>
+  /// <param name="video">The input video stream.</param>
+  /// <param name="audio">The output audio stream.</param>
+  /// <returns>A task that represents the asynchronous operation. The task result indicates whether the extraction was successful.</returns>
+  Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio);
+}

src/StreamShorts.Library/Media/NAudioService.cs

@@ -0,0 +1,45 @@
+
+using NAudio.Wave;
+
+namespace StreamShorts.Library.Media;
+
+/// <summary>
+/// Represents a service for processing audio files using NAudio.
+/// </summary>
+/// <inheritdoc/>
+internal class NAudioService : IAudioService
+{
+  public Stream ConvertMp3ToWav16(Stream mp3)
+  {
+    using var reader = new Mp3FileReader(mp3);
+    var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels);
+    using var resampler = new MediaFoundationResampler(reader, outFormat);
+    var waveStream = new MemoryStream();
+    WaveFileWriter.WriteWavFileToStream(waveStream, resampler);
+    waveStream.Position = 0;
+    return waveStream;
+  }
+
+  public int GetNumberOfWavSegments(Stream wavStream, TimeSpan segmentDuration)
+  {
+    using var waveReader = new WaveFileReader(wavStream);
+    var totalDuration = waveReader.TotalTime;
+    var segmentCount = (int)Math.Ceiling(totalDuration.TotalMilliseconds / segmentDuration.TotalMilliseconds);
+    wavStream.Position = 0;
+    return segmentCount;
+  }
+
+  public Stream GetWavSegment(Stream wavStream, int segmentNumber, TimeSpan segmentDuration)
+  {
+    using var segmentWaveReader = new WaveFileReader(wavStream);
+    var segment = segmentWaveReader.ToSampleProvider()
+      .Skip(segmentNumber * segmentDuration)
+      .Take(segmentDuration);
+    var segmentProvider = segment.ToWaveProvider16();
+    var segmentStream = new MemoryStream();
+    WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider);
+    segmentStream.Position = 0;
+    wavStream.Position = 0;
+    return segmentStream;
+  }
+}

src/StreamShorts.Library/StreamShorts.Library.csproj

@@ -2,5 +2,6 @@
   <ItemGroup>
     <PackageReference Include="FFMpegCore" />
     <PackageReference Include="NAudio" />
+    <PackageReference Include="Whisper.net.AllRuntimes" />
   </ItemGroup>
 </Project>

src/StreamShorts.Library/Transcription/ITranscriber.cs

@@ -1,12 +1,17 @@
+using System.Runtime.CompilerServices;
+
 namespace StreamShorts.Library.Transcription;
 
+/// <summary>
+/// Represents a transcriber interface for audio transcription.
+/// </summary>
 public interface ITranscriber
 {
-  IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(Stream audio);
+  /// <summary>
+  /// Transcribes the audio stream into text segments.
+  /// </summary>
+  /// <param name="audio">The audio stream to transcribe.</param>
+  /// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
+  /// <returns>An <see cref="IAsyncEnumerable{T}"/> where T is <see cref="TranscriptionSegment"/>.</returns>
+  IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(Stream audio, CancellationToken cancellationToken = default);
 }
-
-public record TranscriptionSegment(
-  TimeSpan StartTime,
-  TimeSpan EndTime,
-  string Text
-);

src/StreamShorts.Library/Transcription/TranscriptionSegment.cs

@@ -0,0 +1,13 @@
+namespace StreamShorts.Library.Transcription;
+
+/// <summary>
+/// Represents a segment of transcribed audio.
+/// </summary>
+/// <param name="StartTime">The start time of the segment.</param>
+/// <param name="EndTime">The end time of the segment.</param>
+/// <param name="Text">The transcribed text of the segment.</param>
+public record TranscriptionSegment(
+  TimeSpan StartTime,
+  TimeSpan EndTime,
+  string Text
+);

src/StreamShorts.Library/Transcription/WhisperTranscriber.cs

@@ -1,10 +1,79 @@
+using System.Runtime.CompilerServices;
+
+using StreamShorts.Library.Media;
+
+using Whisper.net;
+using Whisper.net.Ggml;
 
 namespace StreamShorts.Library.Transcription;
 
+/// <summary>
+/// Represents a transcriber that uses Whisper for audio transcription.
+/// </summary>
+/// <inheritdoc/>
 public class WhisperTranscriber : ITranscriber
 {
-  public IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(Stream audio)
+  private readonly IAudioService _audioService = new NAudioService();
+  private WhisperProcessor? _whisperProcessor;
+
+  /// <summary>
+  /// Initializes a new instance of the <see cref="WhisperTranscriber"/> class.
+  /// </summary>
+  public WhisperTranscriber()
   {
-    throw new NotImplementedException();
+  }
+
+  /// <summary>
+  /// Initializes a new instance of the <see cref="WhisperTranscriber"/> class with a specified audio service.
+  /// </summary>
+  /// <param name="audioService">The audio service to use for audio processing.</param>
+  /// <exception cref="ArgumentNullException">Thrown when the audio service is null.</exception
+  internal WhisperTranscriber(IAudioService audioService)
+  {
+    _audioService = audioService ?? throw new ArgumentNullException(nameof(audioService), $"{nameof(audioService)} cannot be null");
+  }
+
+  public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(Stream audio, [EnumeratorCancellation] CancellationToken cancellationToken)
+  {
+    var segmentDuration = TimeSpan.FromMinutes(2);
+    var wavStream = _audioService.ConvertMp3ToWav16(audio);
+    var numberOfSegments = _audioService.GetNumberOfWavSegments(wavStream, segmentDuration);
+
+    foreach (var segmentNumber in Enumerable.Range(0, numberOfSegments))
+    {
+      var segmentStream = _audioService.GetWavSegment(wavStream, segmentNumber, segmentDuration);
+      var durationOffset = TimeSpan.FromMilliseconds(segmentNumber * segmentDuration.TotalMilliseconds);
+
+      await foreach (var result in ProcessSegmentAsync(segmentStream, cancellationToken).ConfigureAwait(false))
+      {
+        yield return new TranscriptionSegment(
+          result.Start + durationOffset,
+          result.End + durationOffset,
+          result.Text
+        );
+      }
+    }
+  }
+
+  private async IAsyncEnumerable<SegmentData> ProcessSegmentAsync(
+    Stream segmentStream,
+    [EnumeratorCancellation] CancellationToken cancellationToken)
+  {
+    if (_whisperProcessor is null)
+    {
+
+      using var modelMemoryStream = new MemoryStream();
+      var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn, cancellationToken: cancellationToken).ConfigureAwait(false);
+      await model.CopyToAsync(modelMemoryStream, cancellationToken).ConfigureAwait(false);
+      var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray());
+      _whisperProcessor = whisperFactory.CreateBuilder()
+        .WithLanguage("en")
+        .Build();
+    }
+
+    await foreach (var result in _whisperProcessor.ProcessAsync(segmentStream, cancellationToken).ConfigureAwait(false))
+    {
+      yield return result;
+    }
   }
 }

tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioConverterTests.cs

@@ -1,10 +0,0 @@
-namespace StreamShorts.Library.Tests.Unit.Media.Audio;
-
-public class AudioConverterTests
-{
-  [Fact]
-  public void Test()
-  {
-    return;
-  }
-}

tests/StreamShorts.Library.Tests/Unit/Media/Audio/AudioExtractorTests.cs

@@ -6,7 +6,7 @@ namespace StreamShorts.Library.Tests.Unit.Media.Audio;
 
 public class AudioExtractorTests
 {
-  private readonly Mock<IFFMpegService> _mockFfmpegService = new();
+  private readonly Mock<IVideoService> _mockFfmpegService = new();
   private readonly AudioExtractor _sut;
 
   public AudioExtractorTests()
@@ -14,6 +14,14 @@ public class AudioExtractorTests
     _sut = new(_mockFfmpegService.Object);
   }
 
+  [Fact]
+  public void Constructor_WhenCalledWithNullFfmpegService_ItShouldThrow()
+  {
+    var action = () => new AudioExtractor(null!);
+
+    action.Should().Throw<ArgumentNullException>();
+  }
+
   [Fact]
   public async Task ExtractMp3FromMp4Async_WhenVideoIsNull_ItShouldThrow()
   {