From 2e19c247efd184ce6599ead004e6483c55f2f225 Mon Sep 17 00:00:00 2001 From: scheilch Date: Tue, 24 Mar 2026 18:02:00 +0100 Subject: [PATCH 1/2] Fix int32 overflow in QSV rate-control parameter computation (#16376) Fix int32 overflow in QSV rate-control parameter computation --- .../MediaEncoding/EncodingHelper.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 27d235affa..0683c99cc2 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -1578,14 +1578,15 @@ namespace MediaBrowser.Controller.MediaEncoding int bitrate = state.OutputVideoBitrate.Value; - // Bit rate under 1000k is not allowed in h264_qsv + // Bit rate under 1000k is not allowed in h264_qsv. if (string.Equals(videoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase)) { bitrate = Math.Max(bitrate, 1000); } - // Currently use the same buffer size for all encoders - int bufsize = bitrate * 2; + // Currently use the same buffer size for all non-QSV encoders. + // Use long arithmetic to prevent int32 overflow for very high bitrate values. + int bufsize = (int)Math.Min((long)bitrate * 2, int.MaxValue); if (string.Equals(videoCodec, "libsvtav1", StringComparison.OrdinalIgnoreCase)) { @@ -1615,7 +1616,13 @@ namespace MediaBrowser.Controller.MediaEncoding // Set (maxrate == bitrate + 1) to trigger VBR for better bitrate allocation // Set (rc_init_occupancy == 2 * bitrate) and (bufsize == 4 * bitrate) to deal with drastic scene changes - return FormattableString.Invariant($"{mbbrcOpt} -b:v {bitrate} -maxrate {bitrate + 1} -rc_init_occupancy {bitrate * 2} -bufsize {bitrate * 4}"); + // Use long arithmetic and clamp to int.MaxValue to prevent int32 overflow + // (e.g. bitrate * 4 wraps to a negative value for bitrates above ~537 million) + int qsvMaxrate = (int)Math.Min((long)bitrate + 1, int.MaxValue); + int qsvInitOcc = (int)Math.Min((long)bitrate * 2, int.MaxValue); + int qsvBufsize = (int)Math.Min((long)bitrate * 4, int.MaxValue); + + return FormattableString.Invariant($"{mbbrcOpt} -b:v {bitrate} -maxrate {qsvMaxrate} -rc_init_occupancy {qsvInitOcc} -bufsize {qsvBufsize}"); } if (string.Equals(videoCodec, "h264_amf", StringComparison.OrdinalIgnoreCase) From 758ee0af767c0586f83f0b08e9f1f14ef2663b2d Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Sat, 2 May 2026 20:14:51 +0800 Subject: [PATCH 2/2] Use strict QSV CPB size for less powerful H.264 decoder Signed-off-by: nyanmisaka --- .../MediaEncoding/EncodingHelper.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 0683c99cc2..e4a8a4cb5c 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -1614,13 +1614,25 @@ namespace MediaBrowser.Controller.MediaEncoding mbbrcOpt = " -mbbrc 1"; } + // Some less powerful H.264 HW decoders require strict CPB size + // So bufsize optimizations should not be applied to them + int factor = 2; + var codec = state.ActualOutputVideoCodec; + var level = state.GetRequestedLevel(codec); + if (string.Equals(codec, "h264", StringComparison.OrdinalIgnoreCase) + && double.TryParse(level, CultureInfo.InvariantCulture, out double requestedLevel) + && requestedLevel < 51) + { + factor = 1; + } + // Set (maxrate == bitrate + 1) to trigger VBR for better bitrate allocation // Set (rc_init_occupancy == 2 * bitrate) and (bufsize == 4 * bitrate) to deal with drastic scene changes // Use long arithmetic and clamp to int.MaxValue to prevent int32 overflow // (e.g. bitrate * 4 wraps to a negative value for bitrates above ~537 million) int qsvMaxrate = (int)Math.Min((long)bitrate + 1, int.MaxValue); - int qsvInitOcc = (int)Math.Min((long)bitrate * 2, int.MaxValue); - int qsvBufsize = (int)Math.Min((long)bitrate * 4, int.MaxValue); + int qsvInitOcc = (int)Math.Min((long)bitrate * 1 * factor, int.MaxValue); + int qsvBufsize = (int)Math.Min((long)bitrate * 2 * factor, int.MaxValue); return FormattableString.Invariant($"{mbbrcOpt} -b:v {bitrate} -maxrate {qsvMaxrate} -rc_init_occupancy {qsvInitOcc} -bufsize {qsvBufsize}"); }