Stevan/stream-shorts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enable processing of large video files over 2GB

Browse Source 
Update audio extraction to use file-based streaming
instead of in-memory buffers to support processing
video files larger than 2GB.

**Changes:**
- Modified `IAudioExtractor.ExtractMp3FromMp4Async()`
  to accept an output audio stream parameter instead
  of creating a MemoryStream internally
- Updated `AudioExtractor` to write directly to the
  provided audio stream, eliminating the need to load
  entire video into memory
- Enhanced `FFMpegService` to detect FileStream inputs
  and use direct file-to-file processing when possible,
  falling back to pipe-based streaming for other stream types
- Refactored `DefaultCommand` to create output directory
  and audio file stream before extraction, enabling direct
  file-based audio extraction
- Updated all unit and integration tests to accommodate the
  new audio stream parameter

This change prevents `OutOfMemoryException` errors when processing
large video files by streaming data directly to disk rather than
buffering in memory. Additionally, it removes the possibility of
overflowing the 2GB size limit that `MemoryStream` has, which
would result in `IOException` errors when trying to write or
copy to it.
This commit is contained in:
Stevan Freeborn
2025-10-10 16:23:02 -05:00
parent 6dcffa6b75
commit e611522c7b
6 changed files with 57 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/StreamShorts.Console/Commands/DefaultCommand.cs
+18 -11
View File
@@ -69,13 +69,28 @@ internal sealed class DefaultCommand(
public override async Task<int> ExecuteAsync(CommandContext context, Settings settings)
{
_console.MarkupLine($"[blue]Processing stream:[/] {settings.Stream.EscapeMarkup()}");
var videoStream = _fileSystem.File.OpenRead(settings.Stream);
var videoStream = (FileStream)_fileSystem.File.OpenRead(settings.Stream);
Stream? audioStream = null;
var now = _timeProvider.GetUtcNow();
var inputFileName = _fileSystem.Path.GetFileNameWithoutExtension(settings.Stream);
var baseDirectory = ValidateAndGetBaseOutputDirectory();
var artifactsOutputDirectory = $"{now:yyyy_MM_dd_HH_mm_ss}_{inputFileName}";
var outputDirectoryPath = _fileSystem.Path.Combine(baseDirectory, artifactsOutputDirectory);
_fileSystem.Directory.CreateDirectory(outputDirectoryPath);
using var audioStream = new FileStream(
_fileSystem.Path.Combine(outputDirectoryPath, "audio.mp3"),
FileMode.Create,
FileAccess.ReadWrite,
FileShare.ReadWrite,
4096,
FileOptions.Asynchronous
);
await _console.Status()
.Spinner(Spinner.Known.Dots)
.StartAsync("Extracting audio...", async _ => audioStream = await _audioExtractor.ExtractMp3FromMp4Async(videoStream));
.StartAsync("Extracting audio...", async _ => await _audioExtractor.ExtractMp3FromMp4Async(videoStream, audioStream));
if (audioStream is null)
{
@@ -101,14 +116,6 @@ internal sealed class DefaultCommand(
_console.MarkupLine($"[blue]Transcription completed[/] [green]successfully![/]");
var now = _timeProvider.GetUtcNow();
var inputFileName = _fileSystem.Path.GetFileNameWithoutExtension(settings.Stream);
var baseDirectory = ValidateAndGetBaseOutputDirectory();
var artifactsOutputDirectory = $"{now:yyyy_MM_dd_HH_mm_ss}_{inputFileName}";
var outputDirectoryPath = _fileSystem.Path.Combine(baseDirectory, artifactsOutputDirectory);
_fileSystem.Directory.CreateDirectory(outputDirectoryPath);
await _fileSystem.File.WriteAllTextAsync(
_fileSystem.Path.Combine(outputDirectoryPath, "transcription.txt"),
string.Join(Environment.NewLine, transcriptionSegments)