feat: initial whipser transcriber implementation

src/StreamShorts.Library/Media/Audio/AudioConverter.cs

@@ -1,18 +0,0 @@
-
-using NAudio.Wave;
-
-namespace StreamShorts.Library.Media.Audio;
-
-public class AudioConverter : IAudioConverter
-{
-  public Stream ConvertMp3ToWav16(Stream mp3)
-  {
-    using var reader = new Mp3FileReader(mp3);
-    var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels);
-    using var resampler = new MediaFoundationResampler(reader, outFormat);
-    var waveStream = new MemoryStream();
-    WaveFileWriter.WriteWavFileToStream(waveStream, resampler);
-    waveStream.Position = 0;
-    return waveStream;
-  }
-}

src/StreamShorts.Library/Media/Audio/AudioExtractor.cs

@@ -1,17 +1,28 @@
 namespace StreamShorts.Library.Media.Audio;
 
+/// <summary>
+/// Extracts audio from video files.
+/// </summary>
 /// <inheritdoc/>
 public class AudioExtractor : IAudioExtractor
 {
-  private readonly IFFMpegService _ffmpegService = new FFMpegService();
+  private readonly IVideoService _videoService = new FFMpegService();
 
+  /// <summary>
+  /// Initializes a new instance of the <see cref="AudioExtractor"/> class.
+  /// </summary>
   public AudioExtractor()
   {
   }
 
-  internal AudioExtractor(IFFMpegService ffmpegService)
+  /// <summary>
+  /// Initializes a new instance of the <see cref="AudioExtractor"/> class with a specified FFMpeg service.
+  /// </summary>
+  /// <param name="videoService">The video service to use for audio extraction.</param>
+  /// <exception cref="ArgumentNullException">Thrown when the FFMpeg service is null.</exception>
+  internal AudioExtractor(IVideoService videoService)
   {
-    _ffmpegService = ffmpegService;
+    _videoService = videoService ?? throw new ArgumentNullException(nameof(videoService), $"{nameof(videoService)} cannot be null");
   }
 
   public async Task<Stream> ExtractMp3FromMp4Async(Stream video)
@@ -41,7 +52,7 @@ public class AudioExtractor : IAudioExtractor
       await video.CopyToAsync(mp4Stream).ConfigureAwait(false);
       mp4Stream.Position = 0;
 
-      var wasExtracted = await _ffmpegService.ExtractAudioFromVideoAsync(mp4Stream, mp3Stream).ConfigureAwait(false);
+      var wasExtracted = await _videoService.ExtractAudioFromVideoAsync(mp4Stream, mp3Stream).ConfigureAwait(false);
 
       if (wasExtracted is false)
       {

src/StreamShorts.Library/Media/Audio/IAudioConverter.cs

@@ -1,6 +0,0 @@
-namespace StreamShorts.Library.Media.Audio;
-
-public interface IAudioConverter
-{
-  Stream ConvertMp3ToWav16(Stream mp3);
-}

src/StreamShorts.Library/Media/FFMpegService.cs

@@ -6,8 +6,11 @@ using FFMpegCore.Pipes;
 
 namespace StreamShorts.Library.Media;
 
-[ExcludeFromCodeCoverage]
-internal class FFMpegService : IFFMpegService
+/// <summary>
+/// Represents a service for processing video files using FFMpeg.
+/// </summary>
+/// <inheritdoc/>
+internal class FFMpegService : IVideoService
 {
   public async Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio)
   {

src/StreamShorts.Library/Media/IAudioService.cs

@@ -0,0 +1,32 @@
+
+namespace StreamShorts.Library.Media;
+
+/// <summary>
+/// Represents a service for processing audio files.
+/// </summary>
+internal interface IAudioService
+{
+  /// <summary>
+  /// Converts an MP3 stream to a WAV stream with a 16 kHz sample rate.
+  /// </summary>
+  /// <param name="mp3">The input MP3 stream.</param>
+  /// <returns>The output WAV stream.</returns>
+  Stream ConvertMp3ToWav16(Stream mp3);
+
+  /// <summary>
+  /// Gets the number of segments in a WAV stream based on the specified segment duration.
+  /// </summary>
+  /// param name="wavStream">The input WAV stream.</param>
+  /// <param name="segmentDuration">The duration of each segment.</param>
+  /// <returns>The number of segments.</returns>
+  int GetNumberOfWavSegments(Stream wavStream, TimeSpan segmentDuration);
+
+  /// <summary>
+  /// Gets a segment of a WAV stream based on the specified segment number and duration.
+  /// </summary>
+  /// param name="wavStream">The input WAV stream.</param>
+  /// <param name="segmentNumber">The segment number to retrieve.</param>
+  /// <param name="segmentDuration">The duration of each segment.</param>
+  /// <returns>The segment stream.</returns>
+  Stream GetWavSegment(Stream wavStream, int segmentNumber, TimeSpan segmentDuration);
+}

src/StreamShorts.Library/Media/IFFMpegService.cs

@@ -1,6 +0,0 @@
-namespace StreamShorts.Library.Media;
-
-internal interface IFFMpegService
-{
-  Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio);
-}

src/StreamShorts.Library/Media/IVideoService.cs

@@ -0,0 +1,15 @@
+namespace StreamShorts.Library.Media;
+
+/// <summary>
+/// Represents a service for processing video files.
+/// </summary>
+internal interface IVideoService
+{
+  /// <summary>
+  /// Extracts audio from a video stream and writes it to an audio stream.
+  /// </summary>
+  /// <param name="video">The input video stream.</param>
+  /// <param name="audio">The output audio stream.</param>
+  /// <returns>A task that represents the asynchronous operation. The task result indicates whether the extraction was successful.</returns>
+  Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio);
+}

src/StreamShorts.Library/Media/NAudioService.cs

@@ -0,0 +1,45 @@
+
+using NAudio.Wave;
+
+namespace StreamShorts.Library.Media;
+
+/// <summary>
+/// Represents a service for processing audio files using NAudio.
+/// </summary>
+/// <inheritdoc/>
+internal class NAudioService : IAudioService
+{
+  public Stream ConvertMp3ToWav16(Stream mp3)
+  {
+    using var reader = new Mp3FileReader(mp3);
+    var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels);
+    using var resampler = new MediaFoundationResampler(reader, outFormat);
+    var waveStream = new MemoryStream();
+    WaveFileWriter.WriteWavFileToStream(waveStream, resampler);
+    waveStream.Position = 0;
+    return waveStream;
+  }
+
+  public int GetNumberOfWavSegments(Stream wavStream, TimeSpan segmentDuration)
+  {
+    using var waveReader = new WaveFileReader(wavStream);
+    var totalDuration = waveReader.TotalTime;
+    var segmentCount = (int)Math.Ceiling(totalDuration.TotalMilliseconds / segmentDuration.TotalMilliseconds);
+    wavStream.Position = 0;
+    return segmentCount;
+  }
+
+  public Stream GetWavSegment(Stream wavStream, int segmentNumber, TimeSpan segmentDuration)
+  {
+    using var segmentWaveReader = new WaveFileReader(wavStream);
+    var segment = segmentWaveReader.ToSampleProvider()
+      .Skip(segmentNumber * segmentDuration)
+      .Take(segmentDuration);
+    var segmentProvider = segment.ToWaveProvider16();
+    var segmentStream = new MemoryStream();
+    WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider);
+    segmentStream.Position = 0;
+    wavStream.Position = 0;
+    return segmentStream;
+  }
+}

src/StreamShorts.Library/StreamShorts.Library.csproj

@@ -2,5 +2,6 @@
   <ItemGroup>
     <PackageReference Include="FFMpegCore" />
     <PackageReference Include="NAudio" />
+    <PackageReference Include="Whisper.net.AllRuntimes" />
   </ItemGroup>
 </Project>

src/StreamShorts.Library/Transcription/ITranscriber.cs

@@ -1,12 +1,17 @@
+using System.Runtime.CompilerServices;
+
 namespace StreamShorts.Library.Transcription;
 
+/// <summary>
+/// Represents a transcriber interface for audio transcription.
+/// </summary>
 public interface ITranscriber
 {
-  IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(Stream audio);
+  /// <summary>
+  /// Transcribes the audio stream into text segments.
+  /// </summary>
+  /// <param name="audio">The audio stream to transcribe.</param>
+  /// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
+  /// <returns>An <see cref="IAsyncEnumerable{T}"/> where T is <see cref="TranscriptionSegment"/>.</returns>
+  IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(Stream audio, CancellationToken cancellationToken = default);
 }
-
-public record TranscriptionSegment(
-  TimeSpan StartTime,
-  TimeSpan EndTime,
-  string Text
-);

src/StreamShorts.Library/Transcription/TranscriptionSegment.cs

@@ -0,0 +1,13 @@
+namespace StreamShorts.Library.Transcription;
+
+/// <summary>
+/// Represents a segment of transcribed audio.
+/// </summary>
+/// <param name="StartTime">The start time of the segment.</param>
+/// <param name="EndTime">The end time of the segment.</param>
+/// <param name="Text">The transcribed text of the segment.</param>
+public record TranscriptionSegment(
+  TimeSpan StartTime,
+  TimeSpan EndTime,
+  string Text
+);

src/StreamShorts.Library/Transcription/WhisperTranscriber.cs

@@ -1,10 +1,79 @@
+using System.Runtime.CompilerServices;
+
+using StreamShorts.Library.Media;
+
+using Whisper.net;
+using Whisper.net.Ggml;
 
 namespace StreamShorts.Library.Transcription;
 
+/// <summary>
+/// Represents a transcriber that uses Whisper for audio transcription.
+/// </summary>
+/// <inheritdoc/>
 public class WhisperTranscriber : ITranscriber
 {
-  public IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(Stream audio)
+  private readonly IAudioService _audioService = new NAudioService();
+  private WhisperProcessor? _whisperProcessor;
+
+  /// <summary>
+  /// Initializes a new instance of the <see cref="WhisperTranscriber"/> class.
+  /// </summary>
+  public WhisperTranscriber()
   {
-    throw new NotImplementedException();
+  }
+
+  /// <summary>
+  /// Initializes a new instance of the <see cref="WhisperTranscriber"/> class with a specified audio service.
+  /// </summary>
+  /// <param name="audioService">The audio service to use for audio processing.</param>
+  /// <exception cref="ArgumentNullException">Thrown when the audio service is null.</exception
+  internal WhisperTranscriber(IAudioService audioService)
+  {
+    _audioService = audioService ?? throw new ArgumentNullException(nameof(audioService), $"{nameof(audioService)} cannot be null");
+  }
+
+  public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(Stream audio, [EnumeratorCancellation] CancellationToken cancellationToken)
+  {
+    var segmentDuration = TimeSpan.FromMinutes(2);
+    var wavStream = _audioService.ConvertMp3ToWav16(audio);
+    var numberOfSegments = _audioService.GetNumberOfWavSegments(wavStream, segmentDuration);
+
+    foreach (var segmentNumber in Enumerable.Range(0, numberOfSegments))
+    {
+      var segmentStream = _audioService.GetWavSegment(wavStream, segmentNumber, segmentDuration);
+      var durationOffset = TimeSpan.FromMilliseconds(segmentNumber * segmentDuration.TotalMilliseconds);
+
+      await foreach (var result in ProcessSegmentAsync(segmentStream, cancellationToken).ConfigureAwait(false))
+      {
+        yield return new TranscriptionSegment(
+          result.Start + durationOffset,
+          result.End + durationOffset,
+          result.Text
+        );
+      }
+    }
+  }
+
+  private async IAsyncEnumerable<SegmentData> ProcessSegmentAsync(
+    Stream segmentStream,
+    [EnumeratorCancellation] CancellationToken cancellationToken)
+  {
+    if (_whisperProcessor is null)
+    {
+
+      using var modelMemoryStream = new MemoryStream();
+      var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn, cancellationToken: cancellationToken).ConfigureAwait(false);
+      await model.CopyToAsync(modelMemoryStream, cancellationToken).ConfigureAwait(false);
+      var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray());
+      _whisperProcessor = whisperFactory.CreateBuilder()
+        .WithLanguage("en")
+        .Build();
+    }
+
+    await foreach (var result in _whisperProcessor.ProcessAsync(segmentStream, cancellationToken).ConfigureAwait(false))
+    {
+      yield return result;
+    }
   }
 }