From b11c8b76dff84bf86f2bc07ee32fccced746a570 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Fri, 12 Sep 2025 00:13:52 +0200
Subject: [PATCH] avcodec/mjpegdec: Avoid mixing return value and error code,
 avoid branch

mjpeg_decode_dc() currently has a special return value (0xfffff)
for the case of invalid data; this value does not overlap with
the ordinary return values, yet the compiler can't prove this
(at least on x86 where an asm version of NEG_USR32 is used
inside get_xbits()), so the non-error return value has to be checked
for being the special value even if mjpeg_decode_dc() is inlined.

This commit avoids this by separating error code and ordinary return
value. It also means that the ljpeg functions return the proper
error code and not -1 in case of invalid data.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/mjpegdec.c | 48 ++++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 23 deletions(-)

diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c
index 8548f6888d..9ee190c419 100644
--- a/libavcodec/mjpegdec.c
+++ b/libavcodec/mjpegdec.c
@@ -815,20 +815,18 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s)
     return 0;
 }
 
-static inline int mjpeg_decode_dc(MJpegDecodeContext *s, int dc_index)
+static inline int mjpeg_decode_dc(MJpegDecodeContext *s, int dc_index, int *val)
 {
     int code;
     code = get_vlc2(&s->gb, s->vlcs[0][dc_index].table, 9, 2);
     if (code < 0 || code > 16) {
         av_log(s->avctx, AV_LOG_ERROR,
                "mjpeg_decode_dc: bad vlc: %d\n", dc_index);
-        return 0xfffff;
+        return AVERROR_INVALIDDATA;
     }
 
-    if (code)
-        return get_xbits(&s->gb, code);
-    else
-        return 0;
+    *val = code ? get_xbits(&s->gb, code) : 0;
+    return 0;
 }
 
 /* decode block and dequantize */
@@ -838,10 +836,10 @@ static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
     int code, i, j, level, val;
 
     /* DC coef */
-    val = mjpeg_decode_dc(s, dc_index);
-    if (val == 0xfffff) {
-        return AVERROR_INVALIDDATA;
-    }
+    int ret = mjpeg_decode_dc(s, dc_index, &val);
+    if (ret < 0)
+        return ret;
+
     val = val * (unsigned)quant_matrix[0] + s->last_dc[component];
     s->last_dc[component] = val;
     block[0] = av_clip_int16(val);
@@ -885,10 +883,10 @@ static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block,
 {
     unsigned val;
     s->bdsp.clear_block(block);
-    val = mjpeg_decode_dc(s, dc_index);
-    if (val == 0xfffff) {
-        return AVERROR_INVALIDDATA;
-    }
+    int ret = mjpeg_decode_dc(s, dc_index, &val);
+    if (ret < 0)
+        return ret;
+
     val = (val * (quant_matrix[0] << Al)) + s->last_dc[component];
     s->last_dc[component] = val;
     block[0] = val;
@@ -1106,6 +1104,7 @@ static int ljpeg_decode_rgb_scan(MJpegDecodeContext *s, int nb_components, int p
     int resync_mb_y = 0;
     int resync_mb_x = 0;
     int vpred[6];
+    int ret;
 
     if (!s->bayer && s->nb_components < 3)
         return AVERROR_INVALIDDATA;
@@ -1178,9 +1177,9 @@ static int ljpeg_decode_rgb_scan(MJpegDecodeContext *s, int nb_components, int p
                 topleft[i] = top[i];
                 top[i]     = buffer[mb_x][i];
 
-                dc = mjpeg_decode_dc(s, s->dc_index[i]);
-                if(dc == 0xFFFFF)
-                    return -1;
+                ret = mjpeg_decode_dc(s, s->dc_index[i], &dc);
+                if (ret < 0)
+                    return ret;
 
                 if (!s->bayer || mb_x) {
                     pred = left[i];
@@ -1274,6 +1273,7 @@ static int ljpeg_decode_yuv_scan(MJpegDecodeContext *s, int predictor,
     int bits= (s->bits+7)&~7;
     int resync_mb_y = 0;
     int resync_mb_x = 0;
+    int ret;
 
     point_transform += bits - s->bits;
     mask = ((1 << s->bits) - 1) << point_transform;
@@ -1312,9 +1312,10 @@ static int ljpeg_decode_yuv_scan(MJpegDecodeContext *s, int predictor,
                     for(j=0; j<n; j++) {
                         int pred, dc;
 
-                        dc = mjpeg_decode_dc(s, s->dc_index[i]);
-                        if(dc == 0xFFFFF)
-                            return -1;
+                        ret = mjpeg_decode_dc(s, s->dc_index[i], &dc);
+                        if (ret < 0)
+                            return ret;
+
                         if (   h * mb_x + x >= s->width
                             || v * mb_y + y >= s->height) {
                             // Nothing to do
@@ -1383,9 +1384,10 @@ static int ljpeg_decode_yuv_scan(MJpegDecodeContext *s, int predictor,
                     for (j = 0; j < n; j++) {
                         int pred;
 
-                        dc = mjpeg_decode_dc(s, s->dc_index[i]);
-                        if(dc == 0xFFFFF)
-                            return -1;
+                        ret = mjpeg_decode_dc(s, s->dc_index[i], &dc);
+                        if (ret < 0)
+                            return ret;
+
                         if (   h * mb_x + x >= s->width
                             || v * mb_y + y >= s->height) {
                             // Nothing to do