feat: refactor audio extraction to reusable class

This commit is contained in:
Stevan Freeborn
2025-07-24 11:22:15 -05:00
parent 310f56fb84
commit 0a661d6970
14 changed files with 354 additions and 181 deletions
+1
View File
@@ -26,6 +26,7 @@ insert_final_newline = false
# Diagnostic severity preferences # Diagnostic severity preferences
dotnet_diagnostic.IDE0058.severity = none dotnet_diagnostic.IDE0058.severity = none
dotnet_diagnostic.IDE0100.severity = none dotnet_diagnostic.IDE0100.severity = none
dotnet_diagnostic.CA1848.severity = none
# Organize usings # Organize usings
dotnet_separate_import_directive_groups = true dotnet_separate_import_directive_groups = true
+1
View File
@@ -7,6 +7,7 @@
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="9.0.7" /> <PackageVersion Include="Microsoft.Extensions.Configuration" Version="9.0.7" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="9.0.7" /> <PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="9.0.7" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.7" /> <PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.7" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.7" />
<PackageVersion Include="NAudio" Version="2.2.1" /> <PackageVersion Include="NAudio" Version="2.2.1" />
<PackageVersion Include="Serilog" Version="4.3.0" /> <PackageVersion Include="Serilog" Version="4.3.0" />
<PackageVersion Include="Serilog.Extensions.Hosting" Version="9.0.0" /> <PackageVersion Include="Serilog.Extensions.Hosting" Version="9.0.0" />
@@ -2,11 +2,13 @@ namespace StreamShorts.Console.Commands;
internal class DefaultCommand( internal class DefaultCommand(
IFileSystem fileSystem, IFileSystem fileSystem,
IAnsiConsole console IAnsiConsole console,
IAudioExtractor audioExtractor
) : AsyncCommand<DefaultCommand.Settings> ) : AsyncCommand<DefaultCommand.Settings>
{ {
private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
private readonly IAnsiConsole _console = console ?? throw new ArgumentNullException(nameof(console)); private readonly IAnsiConsole _console = console ?? throw new ArgumentNullException(nameof(console));
private readonly IAudioExtractor _audioExtractor = audioExtractor ?? throw new ArgumentNullException(nameof(audioExtractor));
internal class Settings : CommandSettings internal class Settings : CommandSettings
{ {
@@ -37,9 +39,28 @@ internal class DefaultCommand(
return base.Validate(context, settings); return base.Validate(context, settings);
} }
public override Task<int> ExecuteAsync(CommandContext context, Settings settings) public override async Task<int> ExecuteAsync(CommandContext context, Settings settings)
{ {
_console.Write($"[green]Processing stream:[/] {settings.Stream}"); _console.MarkupLine($"[blue]Processing stream:[/] {settings.Stream}");
return Task.FromResult(0); var videoStream = _fileSystem.File.OpenRead(settings.Stream);
Stream? audioStream = null;
await _console.Status()
.Spinner(Spinner.Known.Dots)
.StartAsync("Extracting audio...", async ctx =>
{
audioStream = await _audioExtractor.ExtractMp3FromMp4Async(videoStream).ConfigureAwait(false);
});
if (audioStream is null)
{
_console.MarkupLine("[red]Failed[/] to extract audio from the stream.");
return 1;
}
_console.MarkupLine($"[blue]Audio extracted[/] [green]successfully![/]");
return 0;
} }
} }
+176 -174
View File
@@ -10,6 +10,7 @@ using FFMpegCore.Pipes;
using NAudio.Wave; using NAudio.Wave;
using Whisper.net; using Whisper.net;
using Whisper.net.Ggml; using Whisper.net.Ggml;
@@ -35,6 +36,7 @@ try
{ {
services.AddSingleton(AnsiConsole.Console); services.AddSingleton(AnsiConsole.Console);
services.AddSingleton<IFileSystem, FileSystem>(); services.AddSingleton<IFileSystem, FileSystem>();
services.AddSingleton<IAudioExtractor, AudioExtractor>();
}) })
.BuildApp() .BuildApp()
.RunAsync(args); .RunAsync(args);
@@ -51,206 +53,206 @@ finally
await Log.CloseAndFlushAsync(); await Log.CloseAndFlushAsync();
} }
using var mp3Stream = new MemoryStream(); // using var mp3Stream = new MemoryStream();
using var mp4Stream = new FileStream(args[0], FileMode.Open, FileAccess.Read); // using var mp4Stream = new FileStream(args[0], FileMode.Open, FileAccess.Read);
var wasExtracted = await FFMpegArguments // var wasExtracted = await FFMpegArguments
.FromPipeInput(new StreamPipeSource(mp4Stream)) // .FromPipeInput(new StreamPipeSource(mp4Stream))
.OutputToPipe( // .OutputToPipe(
new StreamPipeSink(mp3Stream), // new StreamPipeSink(mp3Stream),
o => o.DisableChannel(Channel.Video).ForceFormat("mp3") // o => o.DisableChannel(Channel.Video).ForceFormat("mp3")
) // )
.ProcessAsynchronously(); // .ProcessAsynchronously();
// Step 2: Convert MP3 stream to 16khz wave format // // Step 2: Convert MP3 stream to 16khz wave format
mp3Stream.Position = 0; // mp3Stream.Position = 0;
using var reader = new Mp3FileReader(mp3Stream); // using var reader = new Mp3FileReader(mp3Stream);
var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels); // var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels);
using var resampler = new MediaFoundationResampler(reader, outFormat); // using var resampler = new MediaFoundationResampler(reader, outFormat);
using var waveStream = new MemoryStream(); // using var waveStream = new MemoryStream();
WaveFileWriter.WriteWavFileToStream(waveStream, resampler); // WaveFileWriter.WriteWavFileToStream(waveStream, resampler);
// Step 3: Split the wave stream into 2 minute segments // // Step 3: Split the wave stream into 2 minute segments
waveStream.Position = 0; // waveStream.Position = 0;
var segmentDuration = TimeSpan.FromMinutes(2); // var segmentDuration = TimeSpan.FromMinutes(2);
var segments = new List<MemoryStream>(); // var segments = new List<MemoryStream>();
using var waveReader = new WaveFileReader(waveStream); // using var waveReader = new WaveFileReader(waveStream);
var segmentCount = (int)Math.Ceiling(waveReader.TotalTime.TotalMilliseconds / segmentDuration.TotalMilliseconds); // var segmentCount = (int)Math.Ceiling(waveReader.TotalTime.TotalMilliseconds / segmentDuration.TotalMilliseconds);
Directory.CreateDirectory("segments"); // Directory.CreateDirectory("segments");
foreach (var i in Enumerable.Range(0, segmentCount)) // foreach (var i in Enumerable.Range(0, segmentCount))
{ // {
waveStream.Position = 0; // waveStream.Position = 0;
using var segmentWaveReader = new WaveFileReader(waveStream); // using var segmentWaveReader = new WaveFileReader(waveStream);
var segment = segmentWaveReader.ToSampleProvider() // var segment = segmentWaveReader.ToSampleProvider()
.Skip(i * segmentDuration) // .Skip(i * segmentDuration)
.Take(segmentDuration); // .Take(segmentDuration);
var segmentProvider = segment.ToWaveProvider16(); // var segmentProvider = segment.ToWaveProvider16();
var segmentStream = new MemoryStream(); // var segmentStream = new MemoryStream();
WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider); // WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider);
segmentStream.Position = 0; // segmentStream.Position = 0;
segments.Add(segmentStream); // segments.Add(segmentStream);
} // }
// Step 4: Transcribe each segment using Whisper // // Step 4: Transcribe each segment using Whisper
using var modelMemoryStream = new MemoryStream(); // using var modelMemoryStream = new MemoryStream();
var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn); // var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn);
await model.CopyToAsync(modelMemoryStream); // await model.CopyToAsync(modelMemoryStream);
var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray()); // var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray());
using var whisperProcessor = whisperFactory.CreateBuilder() // using var whisperProcessor = whisperFactory.CreateBuilder()
.WithLanguage("en") // .WithLanguage("en")
.Build(); // .Build();
var completeTranscription = new StringBuilder(); // var completeTranscription = new StringBuilder();
foreach (var (i, segment) in segments.Select((s, index) => (index, s))) // foreach (var (i, segment) in segments.Select((s, index) => (index, s)))
{ // {
var durationOffset = TimeSpan.FromMilliseconds(i * segmentDuration.TotalMilliseconds); // var durationOffset = TimeSpan.FromMilliseconds(i * segmentDuration.TotalMilliseconds);
var segmentTranscription = new StringBuilder(); // var segmentTranscription = new StringBuilder();
await foreach (var result in whisperProcessor.ProcessAsync(segment, CancellationToken.None)) // await foreach (var result in whisperProcessor.ProcessAsync(segment, CancellationToken.None))
{ // {
var startTime = result.Start + durationOffset; // var startTime = result.Start + durationOffset;
var endTime = result.End + durationOffset; // var endTime = result.End + durationOffset;
segmentTranscription.AppendLine(CultureInfo.CurrentCulture, $"[{startTime:hh\\:mm\\:ss} - {endTime:hh\\:mm\\:ss}] {result.Text}"); // segmentTranscription.AppendLine(CultureInfo.CurrentCulture, $"[{startTime:hh\\:mm\\:ss} - {endTime:hh\\:mm\\:ss}] {result.Text}");
} // }
completeTranscription.Append(segmentTranscription); // completeTranscription.Append(segmentTranscription);
} // }
// Step 5: Send the transcription to LLM for analysis // // Step 5: Send the transcription to LLM for analysis
// TODO: Explore this prompt further...seems break // // TODO: Explore this prompt further...seems break
// when transcription is long // // when transcription is long
var prompt = $$""" // var prompt = $$"""
I need your help to transform my YouTube live stream transcript into engaging YouTube Shorts. Act as my content editor and pinpoint **all potential candidate segments** that are perfect for short-form video. I'm looking for clips that are: // I need your help to transform my YouTube live stream transcript into engaging YouTube Shorts. Act as my content editor and pinpoint **all potential candidate segments** that are perfect for short-form video. I'm looking for clips that are:
  * **Funny:** Moments that will make viewers laugh. //   * **Funny:** Moments that will make viewers laugh.
  * **Informative:** Sections packed with valuable information or tips. //   * **Informative:** Sections packed with valuable information or tips.
  * **Insightful:** Portions offering unique perspectives or 'aha\!' moments. //   * **Insightful:** Portions offering unique perspectives or 'aha\!' moments.
For each suggested short, please provide: // For each suggested short, please provide:
  * The **start time** of the initial segment and the **end time** of the final segment. The duration of each short should be no longer than 3 minutes, but **aim for durations between 15 seconds and 60 seconds**. However, the short **must be as long as necessary to capture the complete thought or idea**, even if it means exceeding the target range or extending slightly to capture all necessary dialogue. //   * The **start time** of the initial segment and the **end time** of the final segment. The duration of each short should be no longer than 3 minutes, but **aim for durations between 15 seconds and 60 seconds**. However, the short **must be as long as necessary to capture the complete thought or idea**, even if it means exceeding the target range or extending slightly to capture all necessary dialogue.
  * A concise **title** that grabs attention. //   * A concise **title** that grabs attention.
  * A brief **description** highlighting the short's content and its appeal. //   * A brief **description** highlighting the short's content and its appeal.
  * An **explanation** of why this particular segment is suitable for a YouTube Short, focusing on its potential for discoverability and engagement. //   * An **explanation** of why this particular segment is suitable for a YouTube Short, focusing on its potential for discoverability and engagement.
Please format your response as a JSON array of objects with the following structure: // Please format your response as a JSON array of objects with the following structure:
```json // ```json
{ // {
  ""title"": ""string"", //   ""title"": ""string"",
  "start_time": "string", //   "start_time": "string",
  "end_time": "string", //   "end_time": "string",
  "description": "string", //   "description": "string",
  "explanation": "string" //   "explanation": "string"
} // }
``` // ```
Here is the transcript of my YouTube live stream: // Here is the transcript of my YouTube live stream:
{{completeTranscription}} // {{completeTranscription}}
"""; // """;
if (string.IsNullOrWhiteSpace(apiKey)) // if (string.IsNullOrWhiteSpace(apiKey))
{ // {
throw new InvalidOperationException("GeminiApiKey is not configured in appsettings.json."); // throw new InvalidOperationException("GeminiApiKey is not configured in appsettings.json.");
} // }
using var client = new HttpClient() // using var client = new HttpClient()
{ // {
Timeout = TimeSpan.FromMinutes(30) // Timeout = TimeSpan.FromMinutes(30)
}; // };
var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={apiKey}"; // var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={apiKey}";
using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl) // using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl)
{ // {
Content = new StringContent( // Content = new StringContent(
JsonSerializer.Serialize(new // JsonSerializer.Serialize(new
{ // {
contents = new[] // contents = new[]
{ // {
new // new
{ // {
role = "user", // role = "user",
parts = new[] // parts = new[]
{ // {
new // new
{ // {
text = prompt // text = prompt
} // }
}, // },
} // }
}, // },
generationConfig = new // generationConfig = new
{ // {
responseMimeType = "application/json", // responseMimeType = "application/json",
} // }
}), // }),
Encoding.UTF8, // Encoding.UTF8,
"application/json" // "application/json"
) // )
}; // };
var response = await client.SendAsync(request); // var response = await client.SendAsync(request);
var responseContent = await response.Content.ReadAsStringAsync(); // var responseContent = await response.Content.ReadAsStringAsync();
var responseJson = JsonSerializer.Deserialize<LLMResponse>(responseContent); // var responseJson = JsonSerializer.Deserialize<LLMResponse>(responseContent);
var candidatesText = responseJson? // var candidatesText = responseJson?
.Candidates? // .Candidates?
.FirstOrDefault()? // .FirstOrDefault()?
.Content // .Content
.Parts?.FirstOrDefault()? // .Parts?.FirstOrDefault()?
.Text; // .Text;
var analysis = JsonSerializer.Deserialize<List<LLMAnalysis>>(candidatesText ?? string.Empty); // var analysis = JsonSerializer.Deserialize<List<LLMAnalysis>>(candidatesText ?? string.Empty);
if (analysis is null) // if (analysis is null)
{ // {
Console.WriteLine(resourceManager.GetString("LLMAnalysisFailed", CultureInfo.CurrentCulture)); // Console.WriteLine(resourceManager.GetString("LLMAnalysisFailed", CultureInfo.CurrentCulture));
return; // return;
} // }
foreach (var result in analysis) // foreach (var result in analysis)
{ // {
var fileName = string.Concat(result.Title.Split(Path.GetInvalidFileNameChars())); // var fileName = string.Concat(result.Title.Split(Path.GetInvalidFileNameChars()));
await FFMpeg.SubVideoAsync( // await FFMpeg.SubVideoAsync(
args[0], // args[0],
$"{fileName}.mp4", // $"{fileName}.mp4",
result.StartTime, // result.StartTime,
result.EndTime // result.EndTime
); // );
} // }
// Step 6: Use analysis to generate a short video // // Step 6: Use analysis to generate a short video
record LLMAnalysis( // record LLMAnalysis(
[property: JsonPropertyName("title")] // [property: JsonPropertyName("title")]
string Title, // string Title,
[property: JsonPropertyName("description")] // [property: JsonPropertyName("description")]
string Description, // string Description,
[property: JsonPropertyName("explanation")] // [property: JsonPropertyName("explanation")]
string Explanation, // string Explanation,
[property: JsonPropertyName("start_time")] // [property: JsonPropertyName("start_time")]
TimeSpan StartTime, // TimeSpan StartTime,
[property: JsonPropertyName("end_time")] // [property: JsonPropertyName("end_time")]
TimeSpan EndTime // TimeSpan EndTime
); // );
record LLMResponse( // record LLMResponse(
[property: JsonPropertyName("candidates")] // [property: JsonPropertyName("candidates")]
Candidate[] Candidates // Candidate[] Candidates
); // );
record Candidate( // record Candidate(
[property: JsonPropertyName("content")] // [property: JsonPropertyName("content")]
Content Content // Content Content
); // );
record Content( // record Content(
[property: JsonPropertyName("parts")] // [property: JsonPropertyName("parts")]
Part[] Parts // Part[] Parts
); // );
record Part( // record Part(
[property: JsonPropertyName("text")] // [property: JsonPropertyName("text")]
string Text // string Text
); // );
@@ -25,4 +25,8 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None> </None>
</ItemGroup> </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StreamShorts.Library\StreamShorts.Library.csproj" />
</ItemGroup>
</Project> </Project>
+1 -1
View File
@@ -2,7 +2,6 @@ global using System.ComponentModel;
global using System.IO.Abstractions; global using System.IO.Abstractions;
global using System.Reflection; global using System.Reflection;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Hosting; global using Microsoft.Extensions.Hosting;
global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Logging;
@@ -16,3 +15,4 @@ global using Spectre.Console.Cli;
global using StreamShorts.Console.Commands; global using StreamShorts.Console.Commands;
global using StreamShorts.Console.Hosting; global using StreamShorts.Console.Hosting;
global using StreamShorts.Library.Media.Audio;
@@ -0,0 +1,63 @@
using FFMpegCore;
using FFMpegCore.Pipes;
using Channel = FFMpegCore.Enums.Channel;
namespace StreamShorts.Library.Media.Audio;
/// <inheritdoc/>
public class AudioExtractor : IAudioExtractor
{
public async Task<Stream> ExtractMp3FromMp4Async(Stream video)
{
if (video is null)
{
throw new ArgumentNullException(nameof(video), "Video stream cannot be null");
}
if (video.CanRead is false)
{
throw new ArgumentException("Video stream must be readable", nameof(video));
}
if (video.CanSeek is false)
{
throw new ArgumentException("Video stream must be seekable", nameof(video));
}
var originalPosition = video.Position;
try
{
using var mp3Stream = new MemoryStream();
using var mp4Stream = new MemoryStream();
await video.CopyToAsync(mp4Stream).ConfigureAwait(false);
mp4Stream.Position = 0;
var wasExtracted = await FFMpegArguments
.FromPipeInput(new StreamPipeSource(mp4Stream))
.OutputToPipe(
new StreamPipeSink(mp3Stream),
o => o.DisableChannel(Channel.Video).ForceFormat("mp3")
)
.ProcessAsynchronously()
.ConfigureAwait(false);
if (wasExtracted is false)
{
throw new FailedAudioExtractionException("Failed to extract audio from the video stream.");
}
return mp3Stream;
}
catch (Exception e) when (e is not FailedAudioExtractionException)
{
throw new FailedAudioExtractionException("Failed to extract audio", e);
}
finally
{
video.Position = originalPosition;
}
}
}
@@ -0,0 +1,16 @@
namespace StreamShorts.Library.Media.Audio;
public class FailedAudioExtractionException : Exception
{
public FailedAudioExtractionException() : base()
{
}
public FailedAudioExtractionException(string message) : base(message)
{
}
public FailedAudioExtractionException(string message, Exception innerException) : base(message, innerException)
{
}
}
@@ -0,0 +1,19 @@
namespace StreamShorts.Library.Media.Audio;
/// <summary>
/// Represents an interface for extracting audio from video streams.
/// </summary>
public interface IAudioExtractor
{
/// <summary>
/// Extracts MP3 audio from an MP4 video stream.
/// </summary>
/// <param name="video">The input video stream.</param>
/// <returns>A stream containing the extracted MP3 audio.</returns>
/// <exception cref="ArgumentNullException">Thrown when the video stream is null.</exception>
/// <exception cref="ArgumentException">Thrown when the video stream is not readable.</exception>
/// <exception cref="ArgumentException">Thrown when the video stream is not seekable.</exception>
/// <exception cref="FailedAudioExtractionException">Thrown when the audio extraction fails.</exception>
/// <remarks>The method will preserve the passed video stream's data and position.</remarks>
Task<Stream> ExtractMp3FromMp4Async(Stream video);
}
@@ -1,3 +1,5 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<PackageReference Include="FFMpegCore" />
</ItemGroup>
</Project> </Project>
+1
View File
@@ -1,2 +1,3 @@
[*.cs] [*.cs]
dotnet_diagnostic.CA1707.severity = none dotnet_diagnostic.CA1707.severity = none
dotnet_diagnostic.CA2007.severity = none
@@ -1,7 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<ItemGroup> <ItemGroup>
<PackageReference Include="AwesomeAssertions" />
<PackageReference Include="Microsoft.NET.Test.Sdk" /> <PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="Moq" />
<PackageReference Include="xunit.v3" /> <PackageReference Include="xunit.v3" />
<PackageReference Include="xunit.runner.visualstudio" /> <PackageReference Include="xunit.runner.visualstudio" />
</ItemGroup> </ItemGroup>
@@ -0,0 +1,36 @@
namespace StreamShorts.Library.Tests.Unit.Media.Audio;
public class AudioExtractorTests
{
private readonly AudioExtractor _sut = new();
[Fact]
public async Task ExtractMp3FromMp4Async_WhenVideoIsNull_ItShouldThrow()
{
var action = async () => await _sut.ExtractMp3FromMp4Async(null!);
await action.Should().ThrowAsync<ArgumentNullException>();
}
[Fact]
public async Task ExtractMp3FromMp4Async_WhenVideoIsNotReadable_ItShouldThrow()
{
var mockStream = new Mock<Stream>();
mockStream.Setup(s => s.CanRead).Returns(false);
var action = async () => await _sut.ExtractMp3FromMp4Async(mockStream.Object);
await action.Should().ThrowAsync<ArgumentException>();
}
[Fact]
public async Task ExtractMp3FromMp4Async_WhenVideoIsNotSeekable_ItShouldThrow()
{
var mockStream = new Mock<Stream>();
mockStream.Setup(s => s.CanSeek).Returns(false);
var action = async () => await _sut.ExtractMp3FromMp4Async(mockStream.Object);
await action.Should().ThrowAsync<ArgumentException>();
}
}
@@ -0,0 +1,5 @@
global using AwesomeAssertions;
global using Moq;
global using StreamShorts.Library.Media.Audio;