avcodec/vaapi_encode: skip AVBR if HRD parameters are set

AVBR does not use VAEncMiscParameterTypeHRD, so attempting to set
rc_buffer_size and bit_rate together will cause the rc_buffer_size
to be ignored if the VAAPI driver supports AVBR. We should prefer
regular VBR for that case.

Signed-off-by: Cameron Gutman <aicommander@gmail.com>
This commit is contained in:
Cameron Gutman 2025-04-01 23:11:20 -05:00 committed by Tong Wu
parent c6297b689f
commit a0936b9769

View file

@ -1294,7 +1294,8 @@ static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
// * If bitrate and quality are both set, try QVBR.
// * If quality is set, try ICQ, then CQP.
// * If bitrate and maxrate are set and have the same value, try CBR.
// * If a bitrate is set, try AVBR, then VBR, then CBR.
// * If bitrate is set and RC buffer size/occupancy is not, try AVBR.
// * If a bitrate is set, try VBR, then CBR.
// * If no bitrate is set, try ICQ, then CQP.
#define TRY_RC_MODE(mode, fail) do { \
@ -1338,7 +1339,10 @@ static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
TRY_RC_MODE(RC_MODE_CBR, 0);
if (avctx->bit_rate > 0) {
TRY_RC_MODE(RC_MODE_AVBR, 0);
// AVBR does not enforce RC buffer constraints
if (!avctx->rc_buffer_size && !avctx->rc_initial_buffer_occupancy)
TRY_RC_MODE(RC_MODE_AVBR, 0);
TRY_RC_MODE(RC_MODE_VBR, 0);
TRY_RC_MODE(RC_MODE_CBR, 0);
} else {