feat: trying to create clips...probs a waste of time grrrrrrr

This commit is contained in:
Stevan Freeborn
2025-08-12 00:17:37 -05:00
parent 8feadcf23c
commit 85304eadd1
12 changed files with 177 additions and 13 deletions
@@ -1,3 +1,8 @@
using System.Globalization;
using System.Text.Json;
using StreamShorts.Library.Media.Video;
namespace StreamShorts.Console.Commands; namespace StreamShorts.Console.Commands;
internal sealed class DefaultCommand( internal sealed class DefaultCommand(
@@ -5,14 +10,24 @@ internal sealed class DefaultCommand(
IAnsiConsole console, IAnsiConsole console,
IAudioExtractor audioExtractor, IAudioExtractor audioExtractor,
ITranscriber transcriber, ITranscriber transcriber,
ITranscriptAnalyzer transcriptAnalyzer ITranscriptAnalyzer transcriptAnalyzer,
IShortsCreator shortsCreator,
TimeProvider timeProvider
) : AsyncCommand<DefaultCommand.Settings> ) : AsyncCommand<DefaultCommand.Settings>
{ {
private readonly JsonSerializerOptions _jsonSerializerOptions = new()
{
WriteIndented = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
};
private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
private readonly IAnsiConsole _console = console ?? throw new ArgumentNullException(nameof(console)); private readonly IAnsiConsole _console = console ?? throw new ArgumentNullException(nameof(console));
private readonly IAudioExtractor _audioExtractor = audioExtractor ?? throw new ArgumentNullException(nameof(audioExtractor)); private readonly IAudioExtractor _audioExtractor = audioExtractor ?? throw new ArgumentNullException(nameof(audioExtractor));
private readonly ITranscriber _transcriber = transcriber ?? throw new ArgumentNullException(nameof(transcriber)); private readonly ITranscriber _transcriber = transcriber ?? throw new ArgumentNullException(nameof(transcriber));
private readonly ITranscriptAnalyzer _transcriptAnalyzer = transcriptAnalyzer ?? throw new ArgumentNullException(nameof(transcriptAnalyzer)); private readonly ITranscriptAnalyzer _transcriptAnalyzer = transcriptAnalyzer ?? throw new ArgumentNullException(nameof(transcriptAnalyzer));
private readonly IShortsCreator _shortsCreator = shortsCreator ?? throw new ArgumentNullException(nameof(shortsCreator));
private readonly TimeProvider _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
internal class Settings : CommandSettings internal class Settings : CommandSettings
{ {
@@ -60,7 +75,7 @@ internal sealed class DefaultCommand(
if (audioStream is null) if (audioStream is null)
{ {
_console.MarkupLine("[red]Failed[/] to extract audio from the stream."); _console.MarkupLine("[red]Failed[/] to extract audio from the stream.");
return 1; return (int)ExitCode.FailedToExtractAudio;
} }
_console.MarkupLine($"[blue]Audio extracted[/] [green]successfully![/]"); _console.MarkupLine($"[blue]Audio extracted[/] [green]successfully![/]");
@@ -90,6 +105,50 @@ internal sealed class DefaultCommand(
analysis = await _transcriptAnalyzer.AnalyzeAsync(transcriptionSegments); analysis = await _transcriptAnalyzer.AnalyzeAsync(transcriptionSegments);
}); });
if (analysis is null)
{
_console.MarkupLine("[red]Failed[/] to analyze transcript.");
return (int)ExitCode.FailedToAnalyzeTranscript;
}
_console.MarkupLine($"[blue]Transcript analysis completed[/] [green]successfully![/]");
var now = _timeProvider.GetUtcNow();
var inputFileName = _fileSystem.Path.GetFileNameWithoutExtension(settings.Stream);
var outputDirectoryPath = _fileSystem.Path.Combine(
AppContext.BaseDirectory,
$"{now:yyyy_MM_dd}_{inputFileName}"
);
var outputDirectory = _fileSystem.Directory.CreateDirectory(outputDirectoryPath);
await _fileSystem.File.WriteAllTextAsync(
_fileSystem.Path.Combine(outputDirectoryPath, "analysis.json"),
JsonSerializer.Serialize(analysis, _jsonSerializerOptions)
);
await _console.Status()
.Spinner(Spinner.Known.Dots)
.StartAsync("Creating shorts...", async ctx =>
{
await foreach (var clip in _shortsCreator.CreateShortsAsync(analysis, videoStream))
{
ctx.Status($"Creating short: {clip.Candidate.Title.EscapeMarkup()}");
var safeFileName = string.Concat(clip.Candidate.Title.Split(_fileSystem.Path.GetInvalidFileNameChars()));
var filePath = _fileSystem.Path.Combine(outputDirectoryPath, $"{safeFileName}.webm");
var fileStream = _fileSystem.File.OpenWrite(filePath);
}
});
_console.MarkupLine($"[blue]Shorts created[/] [green]successfully![/]");
return 0; return 0;
} }
internal enum ExitCode
{
FailedToExtractAudio,
FailedToAnalyzeTranscript,
}
} }
@@ -11,7 +11,7 @@ internal static class HostBuilderExtensions
c.SetExceptionHandler(static (ex, resolver) => c.SetExceptionHandler(static (ex, resolver) =>
{ {
var console = resolver?.Resolve(typeof(IAnsiConsole)) as IAnsiConsole; var console = resolver?.Resolve(typeof(IAnsiConsole)) as IAnsiConsole;
console?.WriteLine($"[red]An error occurred while executing the command:[/]"); console?.MarkupLine($"[red]An error occurred while executing the command:[/]");
console?.WriteException(ex, ExceptionFormats.ShortenEverything); console?.WriteException(ex, ExceptionFormats.ShortenEverything);
}) })
); );
+14 -2
View File
@@ -1,4 +1,8 @@
Log.Logger = new LoggerConfiguration() using Microsoft.Extensions.Configuration;
using StreamShorts.Library.Media.Video;
Log.Logger = new LoggerConfiguration()
.WriteTo.File( .WriteTo.File(
formatter: new CompactJsonFormatter(), formatter: new CompactJsonFormatter(),
path: Path.Combine(AppContext.BaseDirectory, "logs", "log.jsonl"), path: Path.Combine(AppContext.BaseDirectory, "logs", "log.jsonl"),
@@ -18,11 +22,19 @@ try
.ConfigureLogging(static l => l.ClearProviders()) .ConfigureLogging(static l => l.ClearProviders())
.ConfigureServices(static (_, services) => .ConfigureServices(static (_, services) =>
{ {
services.AddHttpClient();
services.AddSingleton(AnsiConsole.Console); services.AddSingleton(AnsiConsole.Console);
services.AddSingleton<IFileSystem, FileSystem>(); services.AddSingleton<IFileSystem, FileSystem>();
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAudioExtractor, AudioExtractor>(); services.AddSingleton<IAudioExtractor, AudioExtractor>();
services.AddSingleton<ITranscriber, WhisperTranscriber>(); services.AddSingleton<ITranscriber, WhisperTranscriber>();
services.AddSingleton<ITranscriptAnalyzer, GeminiAnalyzer>(); services.AddSingleton<ITranscriptAnalyzer, GeminiAnalyzer>(sp =>
{
var clientFactory = sp.GetRequiredService<IHttpClientFactory>();
return new GeminiAnalyzer(clientFactory, "");
});
services.AddSingleton<IShortsCreator, ShortsCreator>();
}) })
.BuildApp() .BuildApp()
.RunAsync(args); .RunAsync(args);
@@ -63,7 +63,7 @@ public sealed class GeminiAnalyzer(
.Parts?.FirstOrDefault()? .Parts?.FirstOrDefault()?
.Text; .Text;
var clips = JsonSerializer.Deserialize<List<ShortClip>>(candidatesText ?? string.Empty); var clips = JsonSerializer.Deserialize<List<ShortCandidate>>(candidatesText ?? string.Empty);
return new TranscriptAnalysis(clips ?? []); return new TranscriptAnalysis(clips ?? []);
} }
} }
@@ -3,9 +3,9 @@ using System.Text.Json.Serialization;
namespace StreamShorts.Library.Analysis; namespace StreamShorts.Library.Analysis;
/// <summary> /// <summary>
/// Represents a short clip derived from a transcript /// Represents a short candidate derived from a transcript.
/// </summary> /// </summary>
public record ShortClip( public record ShortCandidate(
[property: JsonPropertyName("title")] [property: JsonPropertyName("title")]
string Title, string Title,
[property: JsonPropertyName("description")] [property: JsonPropertyName("description")]
@@ -3,10 +3,10 @@ namespace StreamShorts.Library.Analysis;
/// <summary> /// <summary>
/// Represents the analysis of a transcript, containing short clips derived from the transcript. /// Represents the analysis of a transcript, containing short clips derived from the transcript.
/// </summary> /// </summary>
public sealed class TranscriptAnalysis(IEnumerable<ShortClip> shortClips) public sealed class TranscriptAnalysis(IEnumerable<ShortCandidate> candidates)
{ {
/// <summary> /// <summary>
/// Gets the short clips derived from the transcript. /// Gets the short clips derived from the transcript.
/// </summary> /// </summary>
public IEnumerable<ShortClip> ShortClips { get; init; } = shortClips; public IEnumerable<ShortCandidate> Candidates { get; init; } = candidates;
} }
@@ -1,3 +1,4 @@
using FFMpegCore; using FFMpegCore;
using FFMpegCore.Enums; using FFMpegCore.Enums;
using FFMpegCore.Pipes; using FFMpegCore.Pipes;
@@ -21,4 +22,20 @@ internal sealed class FFMpegService : IVideoService
.ProcessAsynchronously() .ProcessAsynchronously()
.ConfigureAwait(false); .ConfigureAwait(false);
} }
public async Task<Stream> ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null)
{
var startTime = start - (buffer ?? TimeSpan.Zero);
var endTime = end + (buffer ?? TimeSpan.Zero);
var outputStream = new MemoryStream();
await FFMpegArguments
.FromPipeInput(new StreamPipeSource(video), o => o.Seek(startTime).EndSeek(endTime))
.OutputToPipe(new StreamPipeSink(outputStream), o => o.CopyChannel().ForceFormat("webm"))
.ProcessAsynchronously()
.ConfigureAwait(false);
outputStream.Position = 0;
return outputStream;
}
} }
@@ -18,7 +18,7 @@ internal interface IAudioService
/// <summary> /// <summary>
/// Gets the number of segments in a WAV stream based on the specified segment duration. /// Gets the number of segments in a WAV stream based on the specified segment duration.
/// </summary> /// </summary>
/// param name="wavStream">The input WAV stream.</param> /// <param name="wavStream">The input WAV stream.</param>
/// <param name="segmentDuration">The duration of each segment.</param> /// <param name="segmentDuration">The duration of each segment.</param>
/// <returns>The number of segments.</returns> /// <returns>The number of segments.</returns>
/// <exception cref="ArgumentNullException">Thrown when the WAV stream is null.</exception> /// <exception cref="ArgumentNullException">Thrown when the WAV stream is null.</exception>
@@ -29,7 +29,7 @@ internal interface IAudioService
/// <summary> /// <summary>
/// Gets a segment of a WAV stream based on the specified segment number and duration. /// Gets a segment of a WAV stream based on the specified segment number and duration.
/// </summary> /// </summary>
/// param name="wavStream">The input WAV stream.</param> /// <param name="wavStream">The input WAV stream.</param>
/// <param name="segmentNumber">The segment number to retrieve.</param> /// <param name="segmentNumber">The segment number to retrieve.</param>
/// <param name="segmentDuration">The duration of each segment.</param> /// <param name="segmentDuration">The duration of each segment.</param>
/// <returns>The segment stream.</returns> /// <returns>The segment stream.</returns>
@@ -12,4 +12,14 @@ internal interface IVideoService
/// <param name="audio">The output audio stream.</param> /// <param name="audio">The output audio stream.</param>
/// <returns>A task that represents the asynchronous operation. The task result indicates whether the extraction was successful.</returns> /// <returns>A task that represents the asynchronous operation. The task result indicates whether the extraction was successful.</returns>
Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio); Task<bool> ExtractAudioFromVideoAsync(Stream video, Stream audio);
/// <summary>
/// Extracts a clip from a video stream.
/// </summary>
/// <param name="video">The input video stream.</param>
/// <param name="start">The start time of the clip.</param>
/// <param name="end">The end time of the clip.</param>
/// <param name="buffer">The duration of the buffer to include before the start time and after the end time.</param>
/// <returns>A task that represents the asynchronous operation. The task result contains the extracted clip as a stream.</returns>
Task<Stream> ExtractClipFromVideoAsync(Stream video, TimeSpan start, TimeSpan end, TimeSpan? buffer = null);
} }
@@ -1,5 +1,20 @@
using StreamShorts.Library.Analysis;
namespace StreamShorts.Library.Media.Video; namespace StreamShorts.Library.Media.Video;
/// <summary>
/// Represents a service that creates video shorts.
/// </summary>
public interface IShortsCreator public interface IShortsCreator
{ {
/// <summary>
/// Creates video shorts from the provided transcript analysis and video stream.
/// </summary>
/// <param name="analysis">The transcript analysis.</param>
/// <param name="video">The video stream.</param>
/// <param name="buffer">An optional buffer duration to include before the start time and after the end time of each short.</param>
/// <returns>An asynchronous enumerable of <see cref="ShortClip"/>.</returns>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="analysis"/> is null.</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="video"/> is null.</exception>
public IAsyncEnumerable<ShortClip> CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null);
} }
@@ -0,0 +1,11 @@
using StreamShorts.Library.Analysis;
namespace StreamShorts.Library.Media.Video;
/// <summary>
/// Represents a short video clip.
/// </summary>
public record ShortClip(
ShortCandidate Candidate,
Stream Segment
);
@@ -1,5 +1,45 @@
using StreamShorts.Library.Analysis;
namespace StreamShorts.Library.Media.Video; namespace StreamShorts.Library.Media.Video;
/// <summary>
/// Represents a service that creates video shorts.
/// </summary>
/// <inheritdoc/>
public class ShortsCreator : IShortsCreator public class ShortsCreator : IShortsCreator
{ {
} private readonly IVideoService _videoService = new FFMpegService();
public ShortsCreator()
{
}
internal ShortsCreator(IVideoService videoService)
{
_videoService = videoService ?? throw new ArgumentNullException(nameof(videoService), $"{nameof(videoService)} cannot be null");
}
public async IAsyncEnumerable<ShortClip> CreateShortsAsync(TranscriptAnalysis analysis, Stream video, TimeSpan? buffer = null)
{
if (analysis is null)
{
throw new ArgumentNullException(nameof(analysis), $"{nameof(analysis)} cannot be null");
}
if (video is null)
{
throw new ArgumentNullException(nameof(video), $"{nameof(video)} cannot be null");
}
foreach (var candidate in analysis.Candidates)
{
var clip = await _videoService.ExtractClipFromVideoAsync(video, candidate.StartTime, candidate.EndTime, buffer)
.ConfigureAwait(false);
yield return new ShortClip(
candidate,
clip
);
}
}
}