swscale: fix signed integer overflow in color conversion arithmetic

Fixes: #22331

Signed-off-by: Hankang Li <hankang201222@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Hankang Li
2026-03-26 03:59:28 -07:00
committed by Michael Niedermayer
parent b47a459867
commit e33b3962e5
2 changed files with 14 additions and 14 deletions

View File

@@ -1180,7 +1180,7 @@ static void planar_rgb_to_y(uint8_t *_dst, const uint8_t *src[4], int width, int
int b = src[1][i];
int r = src[2][i];
dst[i] = (ry*r + gy*g + by*b + (0x801<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6);
dst[i] = (int)((unsigned)ry*r + (unsigned)gy*g + (unsigned)by*b + (0x801<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6);
}
}
@@ -1204,8 +1204,8 @@ static void planar_rgb_to_uv(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[
int b = src[1][i];
int r = src[2][i];
dstU[i] = (ru*r + gu*g + bu*b + (0x4001<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6);
dstV[i] = (rv*r + gv*g + bv*b + (0x4001<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6);
dstU[i] = (int)((unsigned)ru*r + (unsigned)gu*g + (unsigned)bu*b + (0x4001<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6);
dstV[i] = (int)((unsigned)rv*r + (unsigned)gv*g + (unsigned)bv*b + (0x4001<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6);
}
}
@@ -1226,7 +1226,7 @@ static void planar_rgb_to_uv(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[
int b = rdpx(src[1] + i) >> (16 - rdpx_shift); \
int r = rdpx(src[2] + i) >> (16 - rdpx_shift); \
\
dst[i] = (ry*r + gy*g + by*b + (16 << (RGB2YUV_SHIFT + bpc - 8)) \
dst[i] = (int)((unsigned)ry*r + (unsigned)gy*g + (unsigned)by*b + (16 << (RGB2YUV_SHIFT + bpc - 8)) \
+ (1 << (RGB2YUV_SHIFT + shift - 15))) >> (RGB2YUV_SHIFT + shift - 14); \
} \
}
@@ -1262,9 +1262,9 @@ static void planar_rgb_to_uv(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[
int b = rdpx(src[1] + i) >> (16 - rdpx_shift); \
int r = rdpx(src[2] + i) >> (16 - rdpx_shift); \
\
dstU[i] = (ru*r + gu*g + bu*b + (128 << (RGB2YUV_SHIFT + bpc - 8)) \
dstU[i] = (int)((unsigned)ru*r + (unsigned)gu*g + (unsigned)bu*b + (128 << (RGB2YUV_SHIFT + bpc - 8)) \
+ (1 << (RGB2YUV_SHIFT + shift - 15))) >> (RGB2YUV_SHIFT + shift - 14); \
dstV[i] = (rv*r + gv*g + bv*b + (128 << (RGB2YUV_SHIFT + bpc - 8)) \
dstV[i] = (int)((unsigned)rv*r + (unsigned)gv*g + (unsigned)bv*b + (128 << (RGB2YUV_SHIFT + bpc - 8)) \
+ (1 << (RGB2YUV_SHIFT + shift - 15))) >> (RGB2YUV_SHIFT + shift - 14); \
} \
}
@@ -1310,8 +1310,8 @@ static av_always_inline void planar_rgbf32_to_uv(uint8_t *_dstU, uint8_t *_dstV,
int b = lrintf(av_clipf(65535.0f * rdpx(src[1] + i), 0.0f, 65535.0f));
int r = lrintf(av_clipf(65535.0f * rdpx(src[2] + i), 0.0f, 65535.0f));
dstU[i] = (ru*r + gu*g + bu*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
dstV[i] = (rv*r + gv*g + bv*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
dstU[i] = (int)((unsigned)ru*r + (unsigned)gu*g + (unsigned)bu*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
dstV[i] = (int)((unsigned)rv*r + (unsigned)gv*g + (unsigned)bv*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
}
}
@@ -1328,7 +1328,7 @@ static av_always_inline void planar_rgbf32_to_y(uint8_t *_dst, const uint8_t *_s
int b = lrintf(av_clipf(65535.0f * rdpx(src[1] + i), 0.0f, 65535.0f));
int r = lrintf(av_clipf(65535.0f * rdpx(src[2] + i), 0.0f, 65535.0f));
dst[i] = (ry*r + gy*g + by*b + (0x2001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
dst[i] = (int)((unsigned)ry*r + (unsigned)gy*g + (unsigned)by*b + (0x2001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
}
}

View File

@@ -1232,9 +1232,9 @@ yuv2rgba64_2_c_template(SwsInternal *c, const int32_t *buf[2],
Y1 += (1 << 13) - (1 << 29);
Y2 += (1 << 13) - (1 << 29);
R = V * c->yuv2rgb_v2r_coeff;
G = V * c->yuv2rgb_v2g_coeff + U * c->yuv2rgb_u2g_coeff;
B = U * c->yuv2rgb_u2b_coeff;
R = (SUINT)V * c->yuv2rgb_v2r_coeff;
G = (SUINT)V * c->yuv2rgb_v2g_coeff + (SUINT)U * c->yuv2rgb_u2g_coeff;
B = (SUINT)U * c->yuv2rgb_u2b_coeff;
if (hasAlpha) {
A1 = (int)(abuf0[i * 2 ] * yalpha1 + abuf1[i * 2 ] * yalpha) >> 1;
@@ -1327,8 +1327,8 @@ yuv2rgba64_1_c_template(SwsInternal *c, const int32_t *buf0,
for (i = 0; i < ((dstW + 1) >> 1); i++) {
SUINT Y1 = (buf0[i * 2] ) >> 2;
SUINT Y2 = (buf0[i * 2 + 1]) >> 2;
SUINT U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha - (128 << 23)) >> 14;
SUINT V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha - (128 << 23)) >> 14;
SUINT U = (int)((SUINT)ubuf0[i] * uvalpha1 + (SUINT)ubuf1[i] * uvalpha - (128 << 23)) >> 14;
SUINT V = (int)((SUINT)vbuf0[i] * uvalpha1 + (SUINT)vbuf1[i] * uvalpha - (128 << 23)) >> 14;
int R, G, B;
Y1 -= c->yuv2rgb_y_offset;