Merge pull request #11161 from nyanmisaka/fix-segment-deletion

Co-authored-by: Claus Vium <cvium@users.noreply.github.com>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -51,6 +51,7 @@ namespace MediaBrowser.Controller.MediaEncoding
         private readonly Version _minFFmpegOclCuTonemapMode = new Version(5, 1, 3);
         private readonly Version _minFFmpegSvtAv1Params = new Version(5, 1);
         private readonly Version _minFFmpegVaapiH26xEncA53CcSei = new Version(6, 0);
+        private readonly Version _minFFmpegReadrateOption = new Version(5, 0);
 
         private static readonly string[] _videoProfilesH264 = new[]
         {
@@ -1221,7 +1222,7 @@ namespace MediaBrowser.Controller.MediaEncoding
 
             // Disable auto inserted SW scaler for HW decoders in case of changed resolution.
             var isSwDecoder = string.IsNullOrEmpty(GetHardwareVideoDecoder(state, options));
-            if (!isSwDecoder && _mediaEncoder.EncoderVersion >= new Version(4, 4))
+            if (!isSwDecoder)
             {
                 arg.Append(" -noautoscale");
             }
@@ -6393,6 +6394,16 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 inputModifier += " -re";
             }
+            else if (encodingOptions.EnableSegmentDeletion
+                && state.VideoStream is not null
+                && state.TranscodingType == TranscodingJobType.Hls
+                && IsCopyCodec(state.OutputVideoCodec)
+                && _mediaEncoder.EncoderVersion >= _minFFmpegReadrateOption)
+            {
+                // Set an input read rate limit 10x for using SegmentDeletion with stream-copy
+                // to prevent ffmpeg from exiting prematurely (due to fast drive)
+                inputModifier += " -readrate 10";
+            }
 
             var flags = new List<string>();
             if (state.IgnoreInputDts)

MediaBrowser.Controller/MediaEncoding/TranscodingJob.cs

@@ -136,6 +136,11 @@ public sealed class TranscodingJob : IDisposable
     /// </summary>
     public TranscodingThrottler? TranscodingThrottler { get; set; }
 
+    /// <summary>
+    /// Gets or sets transcoding segment cleaner.
+    /// </summary>
+    public TranscodingSegmentCleaner? TranscodingSegmentCleaner { get; set; }
+
     /// <summary>
     /// Gets or sets last ping date.
     /// </summary>
@@ -239,6 +244,7 @@ public sealed class TranscodingJob : IDisposable
         {
 #pragma warning disable CA1849 // Can't await in lock block
             TranscodingThrottler?.Stop().GetAwaiter().GetResult();
+            TranscodingSegmentCleaner?.Stop();
 
             var process = Process;
 
@@ -276,5 +282,7 @@ public sealed class TranscodingJob : IDisposable
         CancellationTokenSource = null;
         TranscodingThrottler?.Dispose();
         TranscodingThrottler = null;
+        TranscodingSegmentCleaner?.Dispose();
+        TranscodingSegmentCleaner = null;
     }
 }

MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs

@@ -0,0 +1,178 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MediaBrowser.Common.Configuration;
+using MediaBrowser.Model.Configuration;
+using MediaBrowser.Model.IO;
+using Microsoft.Extensions.Logging;
+
+namespace MediaBrowser.Controller.MediaEncoding;
+
+/// <summary>
+/// Transcoding segment cleaner.
+/// </summary>
+public class TranscodingSegmentCleaner : IDisposable
+{
+    private readonly TranscodingJob _job;
+    private readonly ILogger<TranscodingSegmentCleaner> _logger;
+    private readonly IConfigurationManager _config;
+    private readonly IFileSystem _fileSystem;
+    private readonly IMediaEncoder _mediaEncoder;
+    private Timer? _timer;
+    private int _segmentLength;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TranscodingSegmentCleaner"/> class.
+    /// </summary>
+    /// <param name="job">Transcoding job dto.</param>
+    /// <param name="logger">Instance of the <see cref="ILogger{TranscodingSegmentCleaner}"/> interface.</param>
+    /// <param name="config">Instance of the <see cref="IConfigurationManager"/> interface.</param>
+    /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param>
+    /// <param name="mediaEncoder">Instance of the <see cref="IMediaEncoder"/> interface.</param>
+    /// <param name="segmentLength">The segment length of this transcoding job.</param>
+    public TranscodingSegmentCleaner(TranscodingJob job, ILogger<TranscodingSegmentCleaner> logger, IConfigurationManager config, IFileSystem fileSystem, IMediaEncoder mediaEncoder, int segmentLength)
+    {
+        _job = job;
+        _logger = logger;
+        _config = config;
+        _fileSystem = fileSystem;
+        _mediaEncoder = mediaEncoder;
+        _segmentLength = segmentLength;
+    }
+
+    /// <summary>
+    /// Start timer.
+    /// </summary>
+    public void Start()
+    {
+        _timer = new Timer(TimerCallback, null, 20000, 20000);
+    }
+
+    /// <summary>
+    /// Stop cleaner.
+    /// </summary>
+    public void Stop()
+    {
+        DisposeTimer();
+    }
+
+    /// <summary>
+    /// Dispose cleaner.
+    /// </summary>
+    public void Dispose()
+    {
+        Dispose(true);
+        GC.SuppressFinalize(this);
+    }
+
+    /// <summary>
+    /// Dispose cleaner.
+    /// </summary>
+    /// <param name="disposing">Disposing.</param>
+    protected virtual void Dispose(bool disposing)
+    {
+        if (disposing)
+        {
+            DisposeTimer();
+        }
+    }
+
+    private EncodingOptions GetOptions()
+    {
+        return _config.GetEncodingOptions();
+    }
+
+    private async void TimerCallback(object? state)
+    {
+        if (_job.HasExited)
+        {
+            DisposeTimer();
+            return;
+        }
+
+        var options = GetOptions();
+        var enableSegmentDeletion = options.EnableSegmentDeletion;
+        var segmentKeepSeconds = Math.Max(options.SegmentKeepSeconds, 20);
+
+        if (enableSegmentDeletion)
+        {
+            var downloadPositionTicks = _job.DownloadPositionTicks ?? 0;
+            var downloadPositionSeconds = Convert.ToInt64(TimeSpan.FromTicks(downloadPositionTicks).TotalSeconds);
+
+            if (downloadPositionSeconds > 0 && segmentKeepSeconds > 0 && downloadPositionSeconds > segmentKeepSeconds)
+            {
+                var idxMaxToDelete = (downloadPositionSeconds - segmentKeepSeconds) / _segmentLength;
+
+                if (idxMaxToDelete > 0)
+                {
+                    await DeleteSegmentFiles(_job, 0, idxMaxToDelete, 1500).ConfigureAwait(false);
+                }
+            }
+        }
+    }
+
+    private async Task DeleteSegmentFiles(TranscodingJob job, long idxMin, long idxMax, int delayMs)
+    {
+        var path = job.Path ?? throw new ArgumentException("Path can't be null.");
+
+        _logger.LogDebug("Deleting segment file(s) index {Min} to {Max} from {Path}", idxMin, idxMax, path);
+
+        await Task.Delay(delayMs).ConfigureAwait(false);
+
+        try
+        {
+            if (job.Type == TranscodingJobType.Hls)
+            {
+                DeleteHlsSegmentFiles(path, idxMin, idxMax);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogDebug(ex, "Error deleting segment file(s) {Path}", path);
+        }
+    }
+
+    private void DeleteHlsSegmentFiles(string outputFilePath, long idxMin, long idxMax)
+    {
+        var directory = Path.GetDirectoryName(outputFilePath)
+                        ?? throw new ArgumentException("Path can't be a root directory.", nameof(outputFilePath));
+
+        var name = Path.GetFileNameWithoutExtension(outputFilePath);
+
+        var filesToDelete = _fileSystem.GetFilePaths(directory)
+            .Where(f => long.TryParse(Path.GetFileNameWithoutExtension(f).Replace(name, string.Empty, StringComparison.Ordinal), out var idx)
+                        && (idx >= idxMin && idx <= idxMax));
+
+        List<Exception>? exs = null;
+        foreach (var file in filesToDelete)
+        {
+            try
+            {
+                _logger.LogDebug("Deleting HLS segment file {0}", file);
+                _fileSystem.DeleteFile(file);
+            }
+            catch (IOException ex)
+            {
+                (exs ??= new List<Exception>()).Add(ex);
+                _logger.LogDebug(ex, "Error deleting HLS segment file {Path}", file);
+            }
+        }
+
+        if (exs is not null)
+        {
+            throw new AggregateException("Error deleting HLS segment files", exs);
+        }
+    }
+
+    private void DisposeTimer()
+    {
+        if (_timer is not null)
+        {
+            _timer.Dispose();
+            _timer = null;
+        }
+    }
+}

MediaBrowser.Controller/MediaEncoding/TranscodingThrottler.cs

@@ -115,7 +115,7 @@ public class TranscodingThrottler : IDisposable
 
         var options = GetOptions();
 
-        if (options.EnableThrottling && IsThrottleAllowed(_job, options.ThrottleDelaySeconds))
+        if (options.EnableThrottling && IsThrottleAllowed(_job, Math.Max(options.ThrottleDelaySeconds, 60)))
         {
             await PauseTranscoding().ConfigureAwait(false);
         }