avcodec/mjpegdec: Avoid mixing return value and error code, avoid branch

mjpeg_decode_dc() currently has a special return value (0xfffff)
for the case of invalid data; this value does not overlap with
the ordinary return values, yet the compiler can't prove this
(at least on x86 where an asm version of NEG_USR32 is used
inside get_xbits()), so the non-error return value has to be checked
for being the special value even if mjpeg_decode_dc() is inlined.

This commit avoids this by separating error code and ordinary return
value. It also means that the ljpeg functions return the proper
error code and not -1 in case of invalid data.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
This commit is contained in:
Andreas Rheinhardt 2025-09-12 00:13:52 +02:00
parent 3b1673db6a
commit b11c8b76df

View file

@ -815,20 +815,18 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s)
return 0;
}
static inline int mjpeg_decode_dc(MJpegDecodeContext *s, int dc_index)
static inline int mjpeg_decode_dc(MJpegDecodeContext *s, int dc_index, int *val)
{
int code;
code = get_vlc2(&s->gb, s->vlcs[0][dc_index].table, 9, 2);
if (code < 0 || code > 16) {
av_log(s->avctx, AV_LOG_ERROR,
"mjpeg_decode_dc: bad vlc: %d\n", dc_index);
return 0xfffff;
return AVERROR_INVALIDDATA;
}
if (code)
return get_xbits(&s->gb, code);
else
return 0;
*val = code ? get_xbits(&s->gb, code) : 0;
return 0;
}
/* decode block and dequantize */
@ -838,10 +836,10 @@ static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
int code, i, j, level, val;
/* DC coef */
val = mjpeg_decode_dc(s, dc_index);
if (val == 0xfffff) {
return AVERROR_INVALIDDATA;
}
int ret = mjpeg_decode_dc(s, dc_index, &val);
if (ret < 0)
return ret;
val = val * (unsigned)quant_matrix[0] + s->last_dc[component];
s->last_dc[component] = val;
block[0] = av_clip_int16(val);
@ -885,10 +883,10 @@ static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block,
{
unsigned val;
s->bdsp.clear_block(block);
val = mjpeg_decode_dc(s, dc_index);
if (val == 0xfffff) {
return AVERROR_INVALIDDATA;
}
int ret = mjpeg_decode_dc(s, dc_index, &val);
if (ret < 0)
return ret;
val = (val * (quant_matrix[0] << Al)) + s->last_dc[component];
s->last_dc[component] = val;
block[0] = val;
@ -1106,6 +1104,7 @@ static int ljpeg_decode_rgb_scan(MJpegDecodeContext *s, int nb_components, int p
int resync_mb_y = 0;
int resync_mb_x = 0;
int vpred[6];
int ret;
if (!s->bayer && s->nb_components < 3)
return AVERROR_INVALIDDATA;
@ -1178,9 +1177,9 @@ static int ljpeg_decode_rgb_scan(MJpegDecodeContext *s, int nb_components, int p
topleft[i] = top[i];
top[i] = buffer[mb_x][i];
dc = mjpeg_decode_dc(s, s->dc_index[i]);
if(dc == 0xFFFFF)
return -1;
ret = mjpeg_decode_dc(s, s->dc_index[i], &dc);
if (ret < 0)
return ret;
if (!s->bayer || mb_x) {
pred = left[i];
@ -1274,6 +1273,7 @@ static int ljpeg_decode_yuv_scan(MJpegDecodeContext *s, int predictor,
int bits= (s->bits+7)&~7;
int resync_mb_y = 0;
int resync_mb_x = 0;
int ret;
point_transform += bits - s->bits;
mask = ((1 << s->bits) - 1) << point_transform;
@ -1312,9 +1312,10 @@ static int ljpeg_decode_yuv_scan(MJpegDecodeContext *s, int predictor,
for(j=0; j<n; j++) {
int pred, dc;
dc = mjpeg_decode_dc(s, s->dc_index[i]);
if(dc == 0xFFFFF)
return -1;
ret = mjpeg_decode_dc(s, s->dc_index[i], &dc);
if (ret < 0)
return ret;
if ( h * mb_x + x >= s->width
|| v * mb_y + y >= s->height) {
// Nothing to do
@ -1383,9 +1384,10 @@ static int ljpeg_decode_yuv_scan(MJpegDecodeContext *s, int predictor,
for (j = 0; j < n; j++) {
int pred;
dc = mjpeg_decode_dc(s, s->dc_index[i]);
if(dc == 0xFFFFF)
return -1;
ret = mjpeg_decode_dc(s, s->dc_index[i], &dc);
if (ret < 0)
return ret;
if ( h * mb_x + x >= s->width
|| v * mb_y + y >= s->height) {
// Nothing to do