swscale/output: Fix integer overflow in yuv2ya16_X_c_template()

Found-by: colod colod <colodcolod7@gmail.com>

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 0c6b7f9483)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2025-10-13 14:32:45 +02:00
parent f39917c627
commit ac4caa33ba
No known key found for this signature in database
GPG key ID: B18E8928B3948D64

View file

@ -943,7 +943,7 @@ yuv2ya16_X_c_template(SwsContext *c, const int16_t *lumFilter,
int A = 0xffff;
for (j = 0; j < lumFilterSize; j++)
Y += lumSrc[j][i] * lumFilter[j];
Y += lumSrc[j][i] * (unsigned)lumFilter[j];
Y >>= 15;
Y += (1<<3) + 0x8000;
@ -952,7 +952,7 @@ yuv2ya16_X_c_template(SwsContext *c, const int16_t *lumFilter,
if (hasAlpha) {
A = -0x40000000 + (1<<14);
for (j = 0; j < lumFilterSize; j++)
A += alpSrc[j][i] * lumFilter[j];
A += alpSrc[j][i] * (unsigned)lumFilter[j];
A >>= 15;
A += 0x8000;