feat: complete implementation using gemini as analyzer
This commit is contained in:
@@ -12,61 +12,68 @@ namespace StreamShorts.Library.Analysis.Gemini;
|
||||
/// <inheritdoc/>
|
||||
public sealed class GeminiAnalyzer(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
string apiKey
|
||||
) : ITranscriptAnalyzer
|
||||
string apiKey,
|
||||
string? model = null
|
||||
) : ITranscriptAnalyzer
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
private readonly IHttpClientFactory _httpClientFactory =
|
||||
httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
|
||||
private readonly string _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey));
|
||||
private readonly string _model = model ?? "gemini-2.5-flash-lite";
|
||||
private readonly IAnalysisPrompt _prompt = new DefaultAnalysisPrompt();
|
||||
|
||||
public GeminiAnalyzer(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
string apiKey,
|
||||
IAnalysisPrompt prompt
|
||||
) : this(httpClientFactory, apiKey)
|
||||
IAnalysisPrompt prompt,
|
||||
string? model = null
|
||||
) : this(httpClientFactory, apiKey, model)
|
||||
{
|
||||
_prompt = prompt ?? throw new ArgumentNullException(nameof(prompt));
|
||||
}
|
||||
|
||||
public async Task<TranscriptAnalysis> AnalyzeAsync(IEnumerable<TranscriptionSegment> segments)
|
||||
{
|
||||
using var client = _httpClientFactory.CreateClient();
|
||||
client.Timeout = TimeSpan.FromMinutes(5);
|
||||
|
||||
var requestUrl = $"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key={_apiKey}";
|
||||
var generateContentRequest = new GenerateContentRequest(
|
||||
[
|
||||
new Content(
|
||||
Role: "user",
|
||||
Parts:[ new Part(Text: _prompt.GetPrompt(segments)) ]
|
||||
)
|
||||
],
|
||||
new GenerationConfig(ResponseMimeType: "application/json")
|
||||
);
|
||||
using var requestContent = new StringContent(
|
||||
JsonSerializer.Serialize(generateContentRequest),
|
||||
Encoding.UTF8,
|
||||
"application/json"
|
||||
);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl)
|
||||
try
|
||||
{
|
||||
Content = requestContent
|
||||
};
|
||||
using var client = _httpClientFactory.CreateClient();
|
||||
client.Timeout = TimeSpan.FromMinutes(5);
|
||||
|
||||
var response = await client.SendAsync(request).ConfigureAwait(false);
|
||||
var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
var responseJson = JsonSerializer.Deserialize<GenerateContentResponse>(responseContent);
|
||||
var candidatesText = responseJson?
|
||||
.Candidates?
|
||||
.FirstOrDefault()?
|
||||
.Content
|
||||
.Parts?.FirstOrDefault()?
|
||||
.Text;
|
||||
var requestUrl =
|
||||
$"https://generativelanguage.googleapis.com/v1beta/models/{_model}:generateContent?key={_apiKey}";
|
||||
var generateContentRequest = new GenerateContentRequest(
|
||||
[
|
||||
new Content(
|
||||
Role: "user",
|
||||
Parts: [new Part(Text: _prompt.GetPrompt(segments))]
|
||||
)
|
||||
],
|
||||
new GenerationConfig(ResponseMimeType: "application/json")
|
||||
);
|
||||
using var requestContent = new StringContent(
|
||||
JsonSerializer.Serialize(generateContentRequest),
|
||||
Encoding.UTF8,
|
||||
"application/json"
|
||||
);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, requestUrl) { Content = requestContent };
|
||||
|
||||
var clips = JsonSerializer.Deserialize<List<ShortCandidate>>(candidatesText ?? string.Empty);
|
||||
return new TranscriptAnalysis(clips ?? []);
|
||||
var response = await client.SendAsync(request).ConfigureAwait(false);
|
||||
var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
var responseJson = JsonSerializer.Deserialize<GenerateContentResponse>(responseContent);
|
||||
var candidatesText = responseJson?
|
||||
.Candidates?
|
||||
.FirstOrDefault()?
|
||||
.Content
|
||||
.Parts?.FirstOrDefault()?
|
||||
.Text;
|
||||
|
||||
var clips = JsonSerializer.Deserialize<List<ShortCandidate>>(candidatesText ?? string.Empty);
|
||||
return new TranscriptAnalysis(clips ?? []);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
throw new FailedTranscriptAnalysisException("Failed to analyze transcript segments using Gemini.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -11,6 +11,7 @@ public interface ITranscriptAnalyzer
|
||||
/// Analyzes the provided transcript segments and generates a transcript analysis result.
|
||||
/// </summary>
|
||||
/// <param name="segments">The transcript segments to analyze.</param>
|
||||
/// <returns>A task that represents the asynchronous operation. The task result contains the <see cref="TranscriptAnalysis"/> containing the short clips derived from the transcript.</returns>
|
||||
/// <returns>A task that represents the asynchronous operation. The task result contains the <see cref="TranscriptAnalysis"/> containing the short clips derived from the transcript.</returns>
|
||||
/// <exception cref="FailedTranscriptAnalysisException">Thrown when the analysis fails due to an error.</exception>
|
||||
Task<TranscriptAnalysis> AnalyzeAsync(IEnumerable<TranscriptionSegment> segments);
|
||||
}
|
||||
@@ -42,8 +42,9 @@ internal sealed class DefaultAnalysisPrompt : IAnalysisPrompt
|
||||
{0}
|
||||
");
|
||||
|
||||
public string GetPrompt(IEnumerable<TranscriptionSegment> transcript)
|
||||
public string GetPrompt(IEnumerable<TranscriptionSegment> segments)
|
||||
{
|
||||
var transcript = string.Join(Environment.NewLine, segments);
|
||||
return string.Format(CultureInfo.InvariantCulture, Prompt, transcript);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
|
||||
using FFMpegCore;
|
||||
using FFMpegCore.Enums;
|
||||
using FFMpegCore.Pipes;
|
||||
@@ -23,19 +22,18 @@ internal sealed class FFMpegService : IVideoService
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<Stream> ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null)
|
||||
public async Task CreateClipFromVideoAsync(
|
||||
string sourcePath,
|
||||
string destinationPath,
|
||||
TimeSpan startTime,
|
||||
TimeSpan endTime,
|
||||
TimeSpan? buffer = null
|
||||
)
|
||||
{
|
||||
var startTime = start - (buffer ?? TimeSpan.Zero);
|
||||
var endTime = end + (buffer ?? TimeSpan.Zero);
|
||||
var outputStream = new MemoryStream();
|
||||
|
||||
await FFMpegArguments
|
||||
.FromPipeInput(new StreamPipeSource(video), o => o.Seek(startTime).EndSeek(endTime))
|
||||
.OutputToPipe(new StreamPipeSink(outputStream), o => o.CopyChannel().ForceFormat("webm"))
|
||||
.ProcessAsynchronously()
|
||||
var start = startTime - (buffer ?? TimeSpan.Zero);
|
||||
var end = endTime + (buffer ?? TimeSpan.Zero);
|
||||
|
||||
await FFMpeg.SubVideoAsync(sourcePath, destinationPath, start, end)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
outputStream.Position = 0;
|
||||
return outputStream;
|
||||
}
|
||||
}
|
||||
@@ -12,14 +12,21 @@ internal interface IVideoService
|
||||
/// <param name="audio">The output audio stream.</param>
|
||||
/// <returns>A task that represents the asynchronous operation. The task result indicates whether the extraction was successful.</returns>
|
||||
Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio);
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Extracts a clip from a video stream.
|
||||
/// Creates a clip from a video file based on the specified start and end times.
|
||||
/// </summary>
|
||||
/// <param name="video">The input video stream.</param>
|
||||
/// <param name="start">The start time of the clip.</param>
|
||||
/// <param name="end">The end time of the clip.</param>
|
||||
/// <param name="buffer">The duration of the buffer to include before the start time and after the end time.</param>
|
||||
/// <returns>A task that represents the asynchronous operation. The task result contains the extracted clip as a stream.</returns>
|
||||
Task<Stream> ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null);
|
||||
/// <param name="sourcePath">The path to the source video file.</param>
|
||||
/// <param name="destinationPath">The path where the created clip will be saved.</param>
|
||||
/// <param name="startTime">The start time of the clip.</param>
|
||||
/// <param name="endTime">The end time of the clip.</param>
|
||||
/// <param name="buffer">An optional buffer time to include before and after the clip segment.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task CreateClipFromVideoAsync(
|
||||
string sourcePath,
|
||||
string destinationPath,
|
||||
TimeSpan startTime,
|
||||
TimeSpan endTime,
|
||||
TimeSpan? buffer = null
|
||||
);
|
||||
}
|
||||
@@ -8,13 +8,14 @@ namespace StreamShorts.Library.Media.Video;
|
||||
public interface IShortsCreator
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates video shorts from the provided transcript analysis and video stream.
|
||||
/// Creates a video short from the specified source video file based on the provided candidate details.
|
||||
/// </summary>
|
||||
/// <param name="analysis">The transcript analysis.</param>
|
||||
/// <param name="video">The video stream.</param>
|
||||
/// <param name="buffer">An optional buffer duration to include before the start time and after the end time of each short.</param>
|
||||
/// <returns>An asynchronous enumerable of <see cref="ShortClip"/>.</returns>
|
||||
/// <exception cref="ArgumentNullException">Thrown when <paramref name="analysis"/> is null.</exception>
|
||||
/// <exception cref="ArgumentNullException">Thrown when <paramref name="video"/> is null.</exception>
|
||||
public IAsyncEnumerable<ShortClip> CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null);
|
||||
/// <param name="sourcePath">The path to the source video file.</param>
|
||||
/// <param name="candidate">The details of the short candidate.</param>
|
||||
/// <param name="destinationPath">The path where the created short will be saved.</param>
|
||||
/// <param name="buffer">An optional buffer time to include before and after the short segment.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
/// <exception cref="ArgumentNullException">Thrown when <paramref name="sourcePath"/> or <paramref name="destinationPath"/> is null or whitespace.</exception>
|
||||
/// <exception cref="ArgumentNullException">Thrown when <paramref name="candidate"/> is null.</exception>
|
||||
public Task CreateShortAsync(string sourcePath, ShortCandidate candidate, string destinationPath, TimeSpan? buffer = null);
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
using StreamShorts.Library.Analysis;
|
||||
|
||||
namespace StreamShorts.Library.Media.Video;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a short video clip.
|
||||
/// </summary>
|
||||
public record ShortClip(
|
||||
ShortCandidate Candidate,
|
||||
Stream Segment
|
||||
);
|
||||
@@ -19,27 +19,24 @@ public class ShortsCreator : IShortsCreator
|
||||
_videoService = videoService ?? throw new ArgumentNullException(nameof(videoService), $"{nameof(videoService)} cannot be null");
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<ShortClip> CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null)
|
||||
public async Task CreateShortAsync(string sourcePath, ShortCandidate candidate, string destinationPath, TimeSpan? buffer = null)
|
||||
{
|
||||
if (analysis is null)
|
||||
if (string.IsNullOrWhiteSpace(sourcePath))
|
||||
{
|
||||
throw new ArgumentNullException(nameof(analysis), $"{nameof(analysis)} cannot be null");
|
||||
throw new ArgumentNullException(nameof(sourcePath), $"{nameof(sourcePath)} cannot be null or whitespace");
|
||||
}
|
||||
|
||||
if (video is null)
|
||||
|
||||
if (candidate is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(video), $"{nameof(video)} cannot be null");
|
||||
throw new ArgumentNullException(nameof(candidate), $"{nameof(candidate)} cannot be null");
|
||||
}
|
||||
|
||||
foreach (var candidate in analysis.Candidates)
|
||||
|
||||
if (string.IsNullOrWhiteSpace(destinationPath))
|
||||
{
|
||||
var clip = await _videoService.ExtractClipFromVideoAsync(video, candidate.StartTime, candidate.EndTime, buffer)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
yield return new ShortClip(
|
||||
candidate,
|
||||
clip
|
||||
);
|
||||
throw new ArgumentNullException(nameof(destinationPath), $"{nameof(destinationPath)} cannot be null or whitespace");
|
||||
}
|
||||
|
||||
await _videoService.CreateClipFromVideoAsync(sourcePath, destinationPath, candidate.StartTime, candidate.EndTime, buffer)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
namespace StreamShorts.Library.Transcription;
|
||||
|
||||
/// <summary>
|
||||
/// Represents an error that occurs when audio transcription fails.
|
||||
/// </summary>
|
||||
public sealed class FailedAudioTranscriptionException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FailedAudioTranscriptionException"/> class with no parameters.
|
||||
/// </summary>
|
||||
public FailedAudioTranscriptionException()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FailedAudioTranscriptionException"/> class with a specified error message.
|
||||
/// </summary>
|
||||
/// <param name="message">The error message that explains the reason for the exception.</param>
|
||||
public FailedAudioTranscriptionException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FailedAudioTranscriptionException"/> class with a specified error message and a reference to the inner exception that is the cause of this exception.
|
||||
/// </summary>
|
||||
/// <param name="message">The error message that explains the reason for the exception.</param>
|
||||
/// <param name="innerException">The exception that is the cause of the current exception.</param>
|
||||
public FailedAudioTranscriptionException(string message, Exception innerException) : base(message, innerException)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -10,4 +10,10 @@ public sealed record TranscriptionSegment(
|
||||
TimeSpan StartTime,
|
||||
TimeSpan EndTime,
|
||||
string Text
|
||||
);
|
||||
)
|
||||
{
|
||||
public override string ToString()
|
||||
{
|
||||
return $"[{StartTime:hh\\:mm\\:ss} - {EndTime:hh\\:mm\\:ss}]: {Text}";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user