avcodec/nvenc: fix vbv buffer size in cq mode

BtbN · BtbN · commit b48951bd29fb · 2022-12-08T12:39:46.000+01:00
The CQ calculation gets thrown off and behaves very nonsensical
if it isn't set to 0.
diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
@@ -1025,8 +1025,9 @@ static av_cold void nvenc_setup_rate_control(AVCodecContext *avctx)
 
         av_log(avctx, AV_LOG_VERBOSE, "CQ(%d) mode enabled.\n", tmp_quality);
 
-        //CQ mode shall discard avg bitrate & honor max bitrate;
+        // CQ mode shall discard avg bitrate/vbv buffer size and honor only max bitrate
         ctx->encode_config.rcParams.averageBitRate = avctx->bit_rate = 0;
+        ctx->encode_config.rcParams.vbvBufferSize = avctx->rc_buffer_size = 0;
         ctx->encode_config.rcParams.maxBitRate = avctx->rc_max_rate;
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -1025,8 +1025,9 @@ static av_cold void nvenc_setup_rate_control(AVCodecContext *avctx)`
`1025`	`1025`
`1026`	`1026`	`av_log(avctx, AV_LOG_VERBOSE, "CQ(%d) mode enabled.\n", tmp_quality);`
`1027`	`1027`
`1028`		`- //CQ mode shall discard avg bitrate & honor max bitrate;`
	`1028`	`+ // CQ mode shall discard avg bitrate/vbv buffer size and honor only max bitrate`
`1029`	`1029`	`ctx->encode_config.rcParams.averageBitRate = avctx->bit_rate = 0;`
	`1030`	`+ ctx->encode_config.rcParams.vbvBufferSize = avctx->rc_buffer_size = 0;`
`1030`	`1031`	`ctx->encode_config.rcParams.maxBitRate = avctx->rc_max_rate;`
`1031`	`1032`	`}`
`1032`	`1033`	`}`