feat: refactor audio extraction to reusable class
This commit is contained in:
Stevan Freeborn
+176
-174
@@ -10,6 +10,7 @@ using FFMpegCore.Pipes;
|
||||
|
||||
using NAudio.Wave;
|
||||
|
||||
|
||||
using Whisper.net;
|
||||
using Whisper.net.Ggml;
|
||||
|
||||
@@ -35,6 +36,7 @@ try
|
||||
{
|
||||
services.AddSingleton(AnsiConsole.Console);
|
||||
services.AddSingleton<IFileSystem, FileSystem>();
|
||||
services.AddSingleton<IAudioExtractor, AudioExtractor>();
|
||||
})
|
||||
.BuildApp()
|
||||
.RunAsync(args);
|
||||
@@ -51,206 +53,206 @@ finally
|
||||
await Log.CloseAndFlushAsync();
|
||||
}
|
||||
|
||||
using var mp3Stream = new MemoryStream();
|
||||
using var mp4Stream = new FileStream(args[0], FileMode.Open, FileAccess.Read);
|
||||
// using var mp3Stream = new MemoryStream();
|
||||
// using var mp4Stream = new FileStream(args[0], FileMode.Open, FileAccess.Read);
|
||||
|
||||
var wasExtracted = await FFMpegArguments
|
||||
.FromPipeInput(new StreamPipeSource(mp4Stream))
|
||||
.OutputToPipe(
|
||||
new StreamPipeSink(mp3Stream),
|
||||
o => o.DisableChannel(Channel.Video).ForceFormat("mp3")
|
||||
)
|
||||
.ProcessAsynchronously();
|
||||
// var wasExtracted = await FFMpegArguments
|
||||
// .FromPipeInput(new StreamPipeSource(mp4Stream))
|
||||
// .OutputToPipe(
|
||||
// new StreamPipeSink(mp3Stream),
|
||||
// o => o.DisableChannel(Channel.Video).ForceFormat("mp3")
|
||||
// )
|
||||
// .ProcessAsynchronously();
|
||||
|
||||
// Step 2: Convert MP3 stream to 16khz wave format
|
||||
mp3Stream.Position = 0;
|
||||
using var reader = new Mp3FileReader(mp3Stream);
|
||||
var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels);
|
||||
using var resampler = new MediaFoundationResampler(reader, outFormat);
|
||||
using var waveStream = new MemoryStream();
|
||||
WaveFileWriter.WriteWavFileToStream(waveStream, resampler);
|
||||
// // Step 2: Convert MP3 stream to 16khz wave format
|
||||
// mp3Stream.Position = 0;
|
||||
// using var reader = new Mp3FileReader(mp3Stream);
|
||||
// var outFormat = new WaveFormat(16000, reader.WaveFormat.Channels);
|
||||
// using var resampler = new MediaFoundationResampler(reader, outFormat);
|
||||
// using var waveStream = new MemoryStream();
|
||||
// WaveFileWriter.WriteWavFileToStream(waveStream, resampler);
|
||||
|
||||
// Step 3: Split the wave stream into 2 minute segments
|
||||
waveStream.Position = 0;
|
||||
var segmentDuration = TimeSpan.FromMinutes(2);
|
||||
var segments = new List<MemoryStream>();
|
||||
using var waveReader = new WaveFileReader(waveStream);
|
||||
var segmentCount = (int)Math.Ceiling(waveReader.TotalTime.TotalMilliseconds / segmentDuration.TotalMilliseconds);
|
||||
// // Step 3: Split the wave stream into 2 minute segments
|
||||
// waveStream.Position = 0;
|
||||
// var segmentDuration = TimeSpan.FromMinutes(2);
|
||||
// var segments = new List<MemoryStream>();
|
||||
// using var waveReader = new WaveFileReader(waveStream);
|
||||
// var segmentCount = (int)Math.Ceiling(waveReader.TotalTime.TotalMilliseconds / segmentDuration.TotalMilliseconds);
|
||||
|
||||
Directory.CreateDirectory("segments");
|
||||
// Directory.CreateDirectory("segments");
|
||||
|
||||
foreach (var i in Enumerable.Range(0, segmentCount))
|
||||
{
|
||||
waveStream.Position = 0;
|
||||
using var segmentWaveReader = new WaveFileReader(waveStream);
|
||||
var segment = segmentWaveReader.ToSampleProvider()
|
||||
.Skip(i * segmentDuration)
|
||||
.Take(segmentDuration);
|
||||
var segmentProvider = segment.ToWaveProvider16();
|
||||
var segmentStream = new MemoryStream();
|
||||
WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider);
|
||||
segmentStream.Position = 0;
|
||||
segments.Add(segmentStream);
|
||||
}
|
||||
// foreach (var i in Enumerable.Range(0, segmentCount))
|
||||
// {
|
||||
// waveStream.Position = 0;
|
||||
// using var segmentWaveReader = new WaveFileReader(waveStream);
|
||||
// var segment = segmentWaveReader.ToSampleProvider()
|
||||
// .Skip(i * segmentDuration)
|
||||
// .Take(segmentDuration);
|
||||
// var segmentProvider = segment.ToWaveProvider16();
|
||||
// var segmentStream = new MemoryStream();
|
||||
// WaveFileWriter.WriteWavFileToStream(segmentStream, segmentProvider);
|
||||
// segmentStream.Position = 0;
|
||||
// segments.Add(segmentStream);
|
||||
// }
|
||||
|
||||
// Step 4: Transcribe each segment using Whisper
|
||||
using var modelMemoryStream = new MemoryStream();
|
||||
var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn);
|
||||
await model.CopyToAsync(modelMemoryStream);
|
||||
var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray());
|
||||
using var whisperProcessor = whisperFactory.CreateBuilder()
|
||||
.WithLanguage("en")
|
||||
.Build();
|
||||
// // Step 4: Transcribe each segment using Whisper
|
||||
// using var modelMemoryStream = new MemoryStream();
|
||||
// var model = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(GgmlType.TinyEn);
|
||||
// await model.CopyToAsync(modelMemoryStream);
|
||||
// var whisperFactory = WhisperFactory.FromBuffer(modelMemoryStream.ToArray());
|
||||
// using var whisperProcessor = whisperFactory.CreateBuilder()
|
||||
// .WithLanguage("en")
|
||||
// .Build();
|
||||
|
||||
var completeTranscription = new StringBuilder();
|
||||
// var completeTranscription = new StringBuilder();
|
||||
|
||||
foreach (var (i, segment) in segments.Select((s, index) => (index, s)))
|
||||
{
|
||||
var durationOffset = TimeSpan.FromMilliseconds(i * segmentDuration.TotalMilliseconds);
|
||||
var segmentTranscription = new StringBuilder();
|
||||
// foreach (var (i, segment) in segments.Select((s, index) => (index, s)))
|
||||
// {
|
||||
// var durationOffset = TimeSpan.FromMilliseconds(i * segmentDuration.TotalMilliseconds);
|
||||
// var segmentTranscription = new StringBuilder();
|
||||
|
||||
await foreach (var result in whisperProcessor.ProcessAsync(segment, CancellationToken.None))
|
||||
{
|
||||
var startTime = result.Start + durationOffset;
|
||||
var endTime = result.End + durationOffset;
|
||||
segmentTranscription.AppendLine(CultureInfo.CurrentCulture, $"[{startTime:hh\\:mm\\:ss} - {endTime:hh\\:mm\\:ss}] {result.Text}");
|
||||
}
|
||||
// await foreach (var result in whisperProcessor.ProcessAsync(segment, CancellationToken.None))
|
||||
// {
|
||||
// var startTime = result.Start + durationOffset;
|
||||
// var endTime = result.End + durationOffset;
|
||||
// segmentTranscription.AppendLine(CultureInfo.CurrentCulture, $"[{startTime:hh\\:mm\\:ss} - {endTime:hh\\:mm\\:ss}] {result.Text}");
|
||||
// }
|
||||
|
||||
completeTranscription.Append(segmentTranscription);
|
||||
}
|
||||
// completeTranscription.Append(segmentTranscription);
|
||||
// }
|
||||
|
||||
// Step 5: Send the transcription to LLM for analysis
|
||||
// TODO: Explore this prompt further...seems break
|
||||
// when transcription is long
|
||||
var prompt = $$"""
|
||||
I need your help to transform my YouTube live stream transcript into engaging YouTube Shorts. Act as my content editor and pinpoint **all potential candidate segments** that are perfect for short-form video. I'm looking for clips that are:
|
||||
// // Step 5: Send the transcription to LLM for analysis
|
||||
// // TODO: Explore this prompt further...seems break
|
||||
// // when transcription is long
|
||||
// var prompt = $$"""
|
||||
// I need your help to transform my YouTube live stream transcript into engaging YouTube Shorts. Act as my content editor and pinpoint **all potential candidate segments** that are perfect for short-form video. I'm looking for clips that are:
|
||||
|
||||
* **Funny:** Moments that will make viewers laugh.
|
||||
* **Informative:** Sections packed with valuable information or tips.
|
||||
* **Insightful:** Portions offering unique perspectives or 'aha\!' moments.
|
||||
// * **Funny:** Moments that will make viewers laugh.
|
||||
// * **Informative:** Sections packed with valuable information or tips.
|
||||
// * **Insightful:** Portions offering unique perspectives or 'aha\!' moments.
|
||||
|
||||
For each suggested short, please provide:
|
||||
// For each suggested short, please provide:
|
||||
|
||||
* The **start time** of the initial segment and the **end time** of the final segment. The duration of each short should be no longer than 3 minutes, but **aim for durations between 15 seconds and 60 seconds**. However, the short **must be as long as necessary to capture the complete thought or idea**, even if it means exceeding the target range or extending slightly to capture all necessary dialogue.
|
||||
* A concise **title** that grabs attention.
|
||||
* A brief **description** highlighting the short's content and its appeal.
|
||||
* An **explanation** of why this particular segment is suitable for a YouTube Short, focusing on its potential for discoverability and engagement.
|
||||
// * The **start time** of the initial segment and the **end time** of the final segment. The duration of each short should be no longer than 3 minutes, but **aim for durations between 15 seconds and 60 seconds**. However, the short **must be as long as necessary to capture the complete thought or idea**, even if it means exceeding the target range or extending slightly to capture all necessary dialogue.
|
||||
// * A concise **title** that grabs attention.
|
||||
// * A brief **description** highlighting the short's content and its appeal.
|
||||
// * An **explanation** of why this particular segment is suitable for a YouTube Short, focusing on its potential for discoverability and engagement.
|
||||
|
||||
Please format your response as a JSON array of objects with the following structure:
|
||||
// Please format your response as a JSON array of objects with the following structure:
|
||||
|
||||
```json
|
||||
{
|
||||
""title"": ""string"",
|
||||
"start_time": "string",
|
||||
"end_time": "string",
|
||||
"description": "string",
|
||||
"explanation": "string"
|
||||
}
|
||||
```
|
||||
// ```json
|
||||
// {
|
||||
// ""title"": ""string"",
|
||||
// "start_time": "string",
|
||||
// "end_time": "string",
|
||||
// "description": "string",
|
||||
// "explanation": "string"
|
||||
// }
|
||||
// ```
|
||||
|
||||
Here is the transcript of my YouTube live stream:
|
||||
// Here is the transcript of my YouTube live stream:
|
||||
|
||||
{{completeTranscription}}
|
||||
""";
|
||||
// {{completeTranscription}}
|
||||
// """;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(apiKey))
|
||||
{
|
||||
throw new InvalidOperationException("GeminiApiKey is not configured in appsettings.json.");
|
||||
}
|
||||
// if (string.IsNullOrWhiteSpace(apiKey))
|
||||
// {
|
||||
// throw new InvalidOperationException("GeminiApiKey is not configured in appsettings.json.");
|
||||
// }
|
||||
|
||||
using var client = new HttpClient()
|
||||
{
|
||||
Timeout = TimeSpan.FromMinutes(30)
|
||||
};
|
||||
// using var client = new HttpClient()
|
||||
// {
|
||||
// Timeout = TimeSpan.FromMinutes(30)
|
||||
// };
|
||||
|
||||
var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={apiKey}";
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl)
|
||||
{
|
||||
Content = new StringContent(
|
||||
JsonSerializer.Serialize(new
|
||||
{
|
||||
contents = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
role = "user",
|
||||
parts = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
text = prompt
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
generationConfig = new
|
||||
{
|
||||
responseMimeType = "application/json",
|
||||
}
|
||||
}),
|
||||
Encoding.UTF8,
|
||||
"application/json"
|
||||
)
|
||||
};
|
||||
var response = await client.SendAsync(request);
|
||||
var responseContent = await response.Content.ReadAsStringAsync();
|
||||
var responseJson = JsonSerializer.Deserialize<LLMResponse>(responseContent);
|
||||
var candidatesText = responseJson?
|
||||
.Candidates?
|
||||
.FirstOrDefault()?
|
||||
.Content
|
||||
.Parts?.FirstOrDefault()?
|
||||
.Text;
|
||||
var analysis = JsonSerializer.Deserialize<List<LLMAnalysis>>(candidatesText ?? string.Empty);
|
||||
// var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={apiKey}";
|
||||
// using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl)
|
||||
// {
|
||||
// Content = new StringContent(
|
||||
// JsonSerializer.Serialize(new
|
||||
// {
|
||||
// contents = new[]
|
||||
// {
|
||||
// new
|
||||
// {
|
||||
// role = "user",
|
||||
// parts = new[]
|
||||
// {
|
||||
// new
|
||||
// {
|
||||
// text = prompt
|
||||
// }
|
||||
// },
|
||||
// }
|
||||
// },
|
||||
// generationConfig = new
|
||||
// {
|
||||
// responseMimeType = "application/json",
|
||||
// }
|
||||
// }),
|
||||
// Encoding.UTF8,
|
||||
// "application/json"
|
||||
// )
|
||||
// };
|
||||
// var response = await client.SendAsync(request);
|
||||
// var responseContent = await response.Content.ReadAsStringAsync();
|
||||
// var responseJson = JsonSerializer.Deserialize<LLMResponse>(responseContent);
|
||||
// var candidatesText = responseJson?
|
||||
// .Candidates?
|
||||
// .FirstOrDefault()?
|
||||
// .Content
|
||||
// .Parts?.FirstOrDefault()?
|
||||
// .Text;
|
||||
// var analysis = JsonSerializer.Deserialize<List<LLMAnalysis>>(candidatesText ?? string.Empty);
|
||||
|
||||
if (analysis is null)
|
||||
{
|
||||
Console.WriteLine(resourceManager.GetString("LLMAnalysisFailed", CultureInfo.CurrentCulture));
|
||||
return;
|
||||
}
|
||||
// if (analysis is null)
|
||||
// {
|
||||
// Console.WriteLine(resourceManager.GetString("LLMAnalysisFailed", CultureInfo.CurrentCulture));
|
||||
// return;
|
||||
// }
|
||||
|
||||
foreach (var result in analysis)
|
||||
{
|
||||
var fileName = string.Concat(result.Title.Split(Path.GetInvalidFileNameChars()));
|
||||
await FFMpeg.SubVideoAsync(
|
||||
args[0],
|
||||
$"{fileName}.mp4",
|
||||
result.StartTime,
|
||||
result.EndTime
|
||||
);
|
||||
}
|
||||
// foreach (var result in analysis)
|
||||
// {
|
||||
// var fileName = string.Concat(result.Title.Split(Path.GetInvalidFileNameChars()));
|
||||
// await FFMpeg.SubVideoAsync(
|
||||
// args[0],
|
||||
// $"{fileName}.mp4",
|
||||
// result.StartTime,
|
||||
// result.EndTime
|
||||
// );
|
||||
// }
|
||||
|
||||
// Step 6: Use analysis to generate a short video
|
||||
// // Step 6: Use analysis to generate a short video
|
||||
|
||||
record LLMAnalysis(
|
||||
[property: JsonPropertyName("title")]
|
||||
string Title,
|
||||
[property: JsonPropertyName("description")]
|
||||
string Description,
|
||||
[property: JsonPropertyName("explanation")]
|
||||
string Explanation,
|
||||
[property: JsonPropertyName("start_time")]
|
||||
TimeSpan StartTime,
|
||||
[property: JsonPropertyName("end_time")]
|
||||
TimeSpan EndTime
|
||||
);
|
||||
// record LLMAnalysis(
|
||||
// [property: JsonPropertyName("title")]
|
||||
// string Title,
|
||||
// [property: JsonPropertyName("description")]
|
||||
// string Description,
|
||||
// [property: JsonPropertyName("explanation")]
|
||||
// string Explanation,
|
||||
// [property: JsonPropertyName("start_time")]
|
||||
// TimeSpan StartTime,
|
||||
// [property: JsonPropertyName("end_time")]
|
||||
// TimeSpan EndTime
|
||||
// );
|
||||
|
||||
record LLMResponse(
|
||||
[property: JsonPropertyName("candidates")]
|
||||
Candidate[] Candidates
|
||||
);
|
||||
// record LLMResponse(
|
||||
// [property: JsonPropertyName("candidates")]
|
||||
// Candidate[] Candidates
|
||||
// );
|
||||
|
||||
record Candidate(
|
||||
[property: JsonPropertyName("content")]
|
||||
Content Content
|
||||
);
|
||||
// record Candidate(
|
||||
// [property: JsonPropertyName("content")]
|
||||
// Content Content
|
||||
// );
|
||||
|
||||
record Content(
|
||||
[property: JsonPropertyName("parts")]
|
||||
Part[] Parts
|
||||
);
|
||||
// record Content(
|
||||
// [property: JsonPropertyName("parts")]
|
||||
// Part[] Parts
|
||||
// );
|
||||
|
||||
record Part(
|
||||
[property: JsonPropertyName("text")]
|
||||
string Text
|
||||
);
|
||||
// record Part(
|
||||
// [property: JsonPropertyName("text")]
|
||||
// string Text
|
||||
// );
|
||||
Reference in New Issue
Block a user