Stevan/stream-shorts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete implementation using gemini as analyzer

Browse Source
This commit is contained in:
Stevan Freeborn
2025-08-19 12:43:34 -05:00
parent 85304eadd1
commit d470574b99
12 changed files with 135 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/StreamShorts.Console/Commands/DefaultCommand.cs
+15 -12
View File
@@ -1,4 +1,3 @@
using System.Globalization;
using System.Text.Json;
using StreamShorts.Library.Media.Video;
@@ -67,7 +66,7 @@ internal sealed class DefaultCommand(
await _console.Status()
.Spinner(Spinner.Known.Dots)
.StartAsync("Extracting audio...", async ctx =>
.StartAsync("Extracting audio...", async _ =>
{
audioStream = await _audioExtractor.ExtractMp3FromMp4Async(videoStream);
});
@@ -96,11 +95,16 @@ internal sealed class DefaultCommand(
_console.MarkupLine($"[blue]Transcription completed[/] [green]successfully![/]");
await _fileSystem.File.WriteAllTextAsync(
_fileSystem.Path.Combine(AppContext.BaseDirectory, "transcription.txt"),
string.Join(Environment.NewLine, transcriptionSegments)
);
TranscriptAnalysis? analysis = null;
await _console.Status()
.Spinner(Spinner.Known.Dots)
.StartAsync("Analyzing transcript...", async ctx =>
.StartAsync("Analyzing transcript...", async _ =>
{
analysis = await _transcriptAnalyzer.AnalyzeAsync(transcriptionSegments);
});
@@ -117,10 +121,10 @@ internal sealed class DefaultCommand(
var inputFileName = _fileSystem.Path.GetFileNameWithoutExtension(settings.Stream);
var outputDirectoryPath = _fileSystem.Path.Combine(
AppContext.BaseDirectory,
$"{now:yyyy_MM_dd}_{inputFileName}"
$"{now:yyyy_MM_dd_HH_mm_ss}_{inputFileName}"
);
var outputDirectory = _fileSystem.Directory.CreateDirectory(outputDirectoryPath);
_fileSystem.Directory.CreateDirectory(outputDirectoryPath);
await _fileSystem.File.WriteAllTextAsync(
_fileSystem.Path.Combine(outputDirectoryPath, "analysis.json"),
@@ -131,13 +135,12 @@ internal sealed class DefaultCommand(
.Spinner(Spinner.Known.Dots)
.StartAsync("Creating shorts...", async ctx =>
{
await foreach (var clip in _shortsCreator.CreateShortsAsync(analysis, videoStream))
foreach (var candidate in analysis.Candidates)
{
ctx.Status($"Creating short: {clip.Candidate.Title.EscapeMarkup()}");
var safeFileName = string.Concat(clip.Candidate.Title.Split(_fileSystem.Path.GetInvalidFileNameChars()));
var filePath = _fileSystem.Path.Combine(outputDirectoryPath, $"{safeFileName}.webm");
var fileStream = _fileSystem.File.OpenWrite(filePath);
ctx.Status($"Creating short: {candidate.Title.EscapeMarkup()}");
var safeFileName = string.Concat(candidate.Title.Split(_fileSystem.Path.GetInvalidFileNameChars()));
var candidatePath = _fileSystem.Path.Combine(outputDirectoryPath, $"{safeFileName}.mp4");
await _shortsCreator.CreateShortAsync(settings.Stream, candidate, candidatePath);
}
});
@@ -146,7 +149,7 @@ internal sealed class DefaultCommand(
return 0;
}
internal enum ExitCode
private enum ExitCode
{
FailedToExtractAudio,
FailedToAnalyzeTranscript,
src/StreamShorts.Console/Program.cs
+14 -1
View File
@@ -20,6 +20,7 @@ try
await Host.CreateDefaultBuilder(args)
.ConfigureLogging(static l => l.ClearProviders())
.ConfigureHostConfiguration(static config => config.AddJsonFile("appsettings.json"))
.ConfigureServices(static (_, services) =>
{
services.AddHttpClient();
@@ -30,9 +31,21 @@ try
services.AddSingleton<ITranscriber, WhisperTranscriber>();
services.AddSingleton<ITranscriptAnalyzer, GeminiAnalyzer>(sp =>
{
const string modelOptionName = "Model";
const string keyOptionName = "ApiKey";
var config = sp.GetRequiredService<IConfiguration>();
var geminiSection = config.GetSection("Gemini");
var key = geminiSection[keyOptionName];
var model = geminiSection[modelOptionName];
if (string.IsNullOrWhiteSpace(key))
{
throw new InvalidOperationException($"{keyOptionName} is not configured in appsettings.json.");
}
var clientFactory = sp.GetRequiredService<IHttpClientFactory>();
return new GeminiAnalyzer(clientFactory, "");
return new GeminiAnalyzer(clientFactory, key, model);
});
services.AddSingleton<IShortsCreator, ShortsCreator>();
})
src/StreamShorts.Library/Analysis/Gemini/GeminiAnalyzer.cs
+19 -12
View File
@@ -12,28 +12,36 @@ namespace StreamShorts.Library.Analysis.Gemini;
/// <inheritdoc/>
public sealed class GeminiAnalyzer(
IHttpClientFactory httpClientFactory,
string apiKey
string apiKey,
string? model = null
) : ITranscriptAnalyzer
{
private readonly IHttpClientFactory _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
private readonly IHttpClientFactory _httpClientFactory =
httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
private readonly string _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey));
private readonly string _model = model ?? "gemini-2.5-flash-lite";
private readonly IAnalysisPrompt _prompt = new DefaultAnalysisPrompt();
public GeminiAnalyzer(
IHttpClientFactory httpClientFactory,
string apiKey,
IAnalysisPrompt prompt
) : this(httpClientFactory, apiKey)
IAnalysisPrompt prompt,
string? model = null
) : this(httpClientFactory, apiKey, model)
{
_prompt = prompt ?? throw new ArgumentNullException(nameof(prompt));
}
public async Task<TranscriptAnalysis> AnalyzeAsync(IEnumerable<TranscriptionSegment> segments)
{
try
{
using var client = _httpClientFactory.CreateClient();
client.Timeout = TimeSpan.FromMinutes(5);
var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={_apiKey}";
var requestUrl =
$"https://generativelanguage.googleapis.com/v1beta/models/{_model}:generateContent?key={_apiKey}";
var generateContentRequest = new GenerateContentRequest(
[
new Content(
@@ -48,10 +56,7 @@ public sealed class GeminiAnalyzer(
Encoding.UTF8,
"application/json"
);
using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl)
{
Content = requestContent
};
using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl) { Content = requestContent };
var response = await client.SendAsync(request).ConfigureAwait(false);
var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
@@ -66,7 +71,9 @@ public sealed class GeminiAnalyzer(
var clips = JsonSerializer.Deserialize<List<ShortCandidate>>(candidatesText ?? string.Empty);
return new TranscriptAnalysis(clips ?? []);
}
catch (Exception e)
{
throw new FailedTranscriptAnalysisException("Failed to analyze transcript segments using Gemini.", e);
}
}
}
src/StreamShorts.Library/Analysis/ITranscriptAnalyzer.cs
+1
View File
@@ -12,5 +12,6 @@ public interface ITranscriptAnalyzer
/// </summary>
/// <param name="segments">The transcript segments to analyze.</param>
/// <returns>A task that represents the asynchronous operation. The task result contains the <see cref="TranscriptAnalysis"/> containing the short clips derived from the transcript.</returns>
/// <exception cref="FailedTranscriptAnalysisException">Thrown when the analysis fails due to an error.</exception>
Task<TranscriptAnalysis> AnalyzeAsync(IEnumerable<TranscriptionSegment> segments);
}
src/StreamShorts.Library/Analysis/Prompts/DefaultAnalysisPrompt.cs
+2 -1
View File
@@ -42,8 +42,9 @@ internal sealed class DefaultAnalysisPrompt : IAnalysisPrompt
{0}
");
public string GetPrompt(IEnumerable<TranscriptionSegment> transcript)
public string GetPrompt(IEnumerable<TranscriptionSegment> segments)
{
var transcript = string.Join(Environment.NewLine, segments);
return string.Format(CultureInfo.InvariantCulture, Prompt, transcript);
}
}
src/StreamShorts.Library/Media/FFMpegService.cs
+10 -12
View File
@@ -1,4 +1,3 @@
using FFMpegCore;
using FFMpegCore.Enums;
using FFMpegCore.Pipes;
@@ -23,19 +22,18 @@ internal sealed class FFMpegService : IVideoService
.ConfigureAwait(false);
}
public async Task<Stream> ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null)
public async Task CreateClipFromVideoAsync(
string sourcePath,
string destinationPath,
TimeSpan startTime,
TimeSpan endTime,
TimeSpan? buffer = null
)
{
var startTime = start - (buffer ?? TimeSpan.Zero);
var endTime = end + (buffer ?? TimeSpan.Zero);
var outputStream = new MemoryStream();
var start = startTime - (buffer ?? TimeSpan.Zero);
var end = endTime + (buffer ?? TimeSpan.Zero);
await FFMpegArguments
.FromPipeInput(new StreamPipeSource(video), o => o.Seek(startTime).EndSeek(endTime))
.OutputToPipe(new StreamPipeSink(outputStream), o => o.CopyChannel().ForceFormat("webm"))
.ProcessAsynchronously()
await FFMpeg.SubVideoAsync(sourcePath, destinationPath, start, end)
.ConfigureAwait(false);
outputStream.Position = 0;
return outputStream;
}
}
src/StreamShorts.Library/Media/IVideoService.cs
+14 -7
View File
@@ -14,12 +14,19 @@ internal interface IVideoService
Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio);
/// <summary>
/// Extracts a clip from a video stream.
/// Creates a clip from a video file based on the specified start and end times.
/// </summary>
/// <param name="video">The input video stream.</param>
/// <param name="start">The start time of the clip.</param>
/// <param name="end">The end time of the clip.</param>
/// <param name="buffer">The duration of the buffer to include before the start time and after the end time.</param>
/// <returns>A task that represents the asynchronous operation. The task result contains the extracted clip as a stream.</returns>
Task<Stream> ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null);
/// <param name="sourcePath">The path to the source video file.</param>
/// <param name="destinationPath">The path where the created clip will be saved.</param>
/// <param name="startTime">The start time of the clip.</param>
/// <param name="endTime">The end time of the clip.</param>
/// <param name="buffer">An optional buffer time to include before and after the clip segment.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
Task CreateClipFromVideoAsync(
string sourcePath,
string destinationPath,
TimeSpan startTime,
TimeSpan endTime,
TimeSpan? buffer = null
);
}
src/StreamShorts.Library/Media/Video/IShortsCreator.cs
+9 -8
View File
@@ -8,13 +8,14 @@ namespace StreamShorts.Library.Media.Video;
public interface IShortsCreator
{
/// <summary>
/// Creates video shorts from the provided transcript analysis and video stream.
/// Creates a video short from the specified source video file based on the provided candidate details.
/// </summary>
/// <param name="analysis">The transcript analysis.</param>
/// <param name="video">The video stream.</param>
/// <param name="buffer">An optional buffer duration to include before the start time and after the end time of each short.</param>
/// <returns>An asynchronous enumerable of <see cref="ShortClip"/>.</returns>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="analysis"/> is null.</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="video"/> is null.</exception>
public IAsyncEnumerable<ShortClip> CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null);
/// <param name="sourcePath">The path to the source video file.</param>
/// <param name="candidate">The details of the short candidate.</param>
/// <param name="destinationPath">The path where the created short will be saved.</param>
/// <param name="buffer">An optional buffer time to include before and after the short segment.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="sourcePath"/> or <paramref name="destinationPath"/> is null or whitespace.</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="candidate"/> is null.</exception>
public Task CreateShortAsync(string sourcePath, ShortCandidate candidate, string destinationPath, TimeSpan? buffer = null);
}
src/StreamShorts.Library/Media/Video/ShortClip.cs
-11
View File
@@ -1,11 +0,0 @@
using StreamShorts.Library.Analysis;
namespace StreamShorts.Library.Media.Video;
/// <summary>
/// Represents a short video clip.
/// </summary>
public record ShortClip(
ShortCandidate Candidate,
Stream Segment
);
src/StreamShorts.Library/Media/Video/ShortsCreator.cs
+10 -13
View File
@@ -19,27 +19,24 @@ public class ShortsCreator : IShortsCreator
_videoService = videoService ?? throw new ArgumentNullException(nameof(videoService), $"{nameof(videoService)} cannot be null");
}
public async IAsyncEnumerable<ShortClip> CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null)
public async Task CreateShortAsync(string sourcePath, ShortCandidate candidate, string destinationPath, TimeSpan? buffer = null)
{
if (analysis is null)
if (string.IsNullOrWhiteSpace(sourcePath))
{
throw new ArgumentNullException(nameof(analysis), $"{nameof(analysis)} cannot be null");
throw new ArgumentNullException(nameof(sourcePath), $"{nameof(sourcePath)} cannot be null or whitespace");
}
if (video is null)
if (candidate is null)
{
throw new ArgumentNullException(nameof(video), $"{nameof(video)} cannot be null");
throw new ArgumentNullException(nameof(candidate), $"{nameof(candidate)} cannot be null");
}
foreach (var candidate in analysis.Candidates)
if (string.IsNullOrWhiteSpace(destinationPath))
{
var clip = await _videoService.ExtractClipFromVideoAsync(video, candidate.StartTime, candidate.EndTime, buffer)
throw new ArgumentNullException(nameof(destinationPath), $"{nameof(destinationPath)} cannot be null or whitespace");
}
await _videoService.CreateClipFromVideoAsync(sourcePath, destinationPath, candidate.StartTime, candidate.EndTime, buffer)
.ConfigureAwait(false);
yield return new ShortClip(
candidate,
clip
);
}
}
}
src/StreamShorts.Library/Transcription/FailedAudioTranscriptionException.cs
-31
View File
@@ -1,31 +0,0 @@
namespace StreamShorts.Library.Transcription;
/// <summary>
/// Represents an error that occurs when audio transcription fails.
/// </summary>
public sealed class FailedAudioTranscriptionException : Exception
{
/// <summary>
/// Initializes a new instance of the <see cref="FailedAudioTranscriptionException"/> class with no parameters.
/// </summary>
public FailedAudioTranscriptionException()
{
}
/// <summary>
/// Initializes a new instance of the <see cref="FailedAudioTranscriptionException"/> class with a specified error message.
/// </summary>
/// <param name="message">The error message that explains the reason for the exception.</param>
public FailedAudioTranscriptionException(string message) : base(message)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="FailedAudioTranscriptionException"/> class with a specified error message and a reference to the inner exception that is the cause of this exception.
/// </summary>
/// <param name="message">The error message that explains the reason for the exception.</param>
/// <param name="innerException">The exception that is the cause of the current exception.</param>
public FailedAudioTranscriptionException(string message, Exception innerException) : base(message, innerException)
{
}
}
src/StreamShorts.Library/Transcription/TranscriptionSegment.cs
+7 -1
View File
@@ -10,4 +10,10 @@ public sealed record TranscriptionSegment(
TimeSpan StartTime,
TimeSpan EndTime,
string Text
);
)
{
public override string ToString()
{
return $"[{StartTime:hh\\:mm\\:ss} - {EndTime:hh\\:mm\\:ss}]: {Text}";
}
}