From e33b3962e5b12cbf3e2d2c6387331c10c6d46bb5 Mon Sep 17 00:00:00 2001 From: Hankang Li Date: Thu, 26 Mar 2026 03:59:28 -0700 Subject: [PATCH] swscale: fix signed integer overflow in color conversion arithmetic Fixes: #22331 Signed-off-by: Hankang Li Signed-off-by: Michael Niedermayer --- libswscale/input.c | 18 +++++++++--------- libswscale/output.c | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/libswscale/input.c b/libswscale/input.c index eb4eb3101e..175ed705f2 100644 --- a/libswscale/input.c +++ b/libswscale/input.c @@ -1180,7 +1180,7 @@ static void planar_rgb_to_y(uint8_t *_dst, const uint8_t *src[4], int width, int int b = src[1][i]; int r = src[2][i]; - dst[i] = (ry*r + gy*g + by*b + (0x801<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6); + dst[i] = (int)((unsigned)ry*r + (unsigned)gy*g + (unsigned)by*b + (0x801<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6); } } @@ -1204,8 +1204,8 @@ static void planar_rgb_to_uv(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[ int b = src[1][i]; int r = src[2][i]; - dstU[i] = (ru*r + gu*g + bu*b + (0x4001<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6); - dstV[i] = (rv*r + gv*g + bv*b + (0x4001<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6); + dstU[i] = (int)((unsigned)ru*r + (unsigned)gu*g + (unsigned)bu*b + (0x4001<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6); + dstV[i] = (int)((unsigned)rv*r + (unsigned)gv*g + (unsigned)bv*b + (0x4001<<(RGB2YUV_SHIFT-7))) >> (RGB2YUV_SHIFT-6); } } @@ -1226,7 +1226,7 @@ static void planar_rgb_to_uv(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[ int b = rdpx(src[1] + i) >> (16 - rdpx_shift); \ int r = rdpx(src[2] + i) >> (16 - rdpx_shift); \ \ - dst[i] = (ry*r + gy*g + by*b + (16 << (RGB2YUV_SHIFT + bpc - 8)) \ + dst[i] = (int)((unsigned)ry*r + (unsigned)gy*g + (unsigned)by*b + (16 << (RGB2YUV_SHIFT + bpc - 8)) \ + (1 << (RGB2YUV_SHIFT + shift - 15))) >> (RGB2YUV_SHIFT + shift - 14); \ } \ } @@ -1262,9 +1262,9 @@ static void planar_rgb_to_uv(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[ int b = rdpx(src[1] + i) >> (16 - rdpx_shift); \ int r = rdpx(src[2] + i) >> (16 - rdpx_shift); \ \ - dstU[i] = (ru*r + gu*g + bu*b + (128 << (RGB2YUV_SHIFT + bpc - 8)) \ + dstU[i] = (int)((unsigned)ru*r + (unsigned)gu*g + (unsigned)bu*b + (128 << (RGB2YUV_SHIFT + bpc - 8)) \ + (1 << (RGB2YUV_SHIFT + shift - 15))) >> (RGB2YUV_SHIFT + shift - 14); \ - dstV[i] = (rv*r + gv*g + bv*b + (128 << (RGB2YUV_SHIFT + bpc - 8)) \ + dstV[i] = (int)((unsigned)rv*r + (unsigned)gv*g + (unsigned)bv*b + (128 << (RGB2YUV_SHIFT + bpc - 8)) \ + (1 << (RGB2YUV_SHIFT + shift - 15))) >> (RGB2YUV_SHIFT + shift - 14); \ } \ } @@ -1310,8 +1310,8 @@ static av_always_inline void planar_rgbf32_to_uv(uint8_t *_dstU, uint8_t *_dstV, int b = lrintf(av_clipf(65535.0f * rdpx(src[1] + i), 0.0f, 65535.0f)); int r = lrintf(av_clipf(65535.0f * rdpx(src[2] + i), 0.0f, 65535.0f)); - dstU[i] = (ru*r + gu*g + bu*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT; - dstV[i] = (rv*r + gv*g + bv*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT; + dstU[i] = (int)((unsigned)ru*r + (unsigned)gu*g + (unsigned)bu*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT; + dstV[i] = (int)((unsigned)rv*r + (unsigned)gv*g + (unsigned)bv*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT; } } @@ -1328,7 +1328,7 @@ static av_always_inline void planar_rgbf32_to_y(uint8_t *_dst, const uint8_t *_s int b = lrintf(av_clipf(65535.0f * rdpx(src[1] + i), 0.0f, 65535.0f)); int r = lrintf(av_clipf(65535.0f * rdpx(src[2] + i), 0.0f, 65535.0f)); - dst[i] = (ry*r + gy*g + by*b + (0x2001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT; + dst[i] = (int)((unsigned)ry*r + (unsigned)gy*g + (unsigned)by*b + (0x2001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT; } } diff --git a/libswscale/output.c b/libswscale/output.c index 1494b06bf5..396380ac72 100644 --- a/libswscale/output.c +++ b/libswscale/output.c @@ -1232,9 +1232,9 @@ yuv2rgba64_2_c_template(SwsInternal *c, const int32_t *buf[2], Y1 += (1 << 13) - (1 << 29); Y2 += (1 << 13) - (1 << 29); - R = V * c->yuv2rgb_v2r_coeff; - G = V * c->yuv2rgb_v2g_coeff + U * c->yuv2rgb_u2g_coeff; - B = U * c->yuv2rgb_u2b_coeff; + R = (SUINT)V * c->yuv2rgb_v2r_coeff; + G = (SUINT)V * c->yuv2rgb_v2g_coeff + (SUINT)U * c->yuv2rgb_u2g_coeff; + B = (SUINT)U * c->yuv2rgb_u2b_coeff; if (hasAlpha) { A1 = (int)(abuf0[i * 2 ] * yalpha1 + abuf1[i * 2 ] * yalpha) >> 1; @@ -1327,8 +1327,8 @@ yuv2rgba64_1_c_template(SwsInternal *c, const int32_t *buf0, for (i = 0; i < ((dstW + 1) >> 1); i++) { SUINT Y1 = (buf0[i * 2] ) >> 2; SUINT Y2 = (buf0[i * 2 + 1]) >> 2; - SUINT U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha - (128 << 23)) >> 14; - SUINT V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha - (128 << 23)) >> 14; + SUINT U = (int)((SUINT)ubuf0[i] * uvalpha1 + (SUINT)ubuf1[i] * uvalpha - (128 << 23)) >> 14; + SUINT V = (int)((SUINT)vbuf0[i] * uvalpha1 + (SUINT)vbuf1[i] * uvalpha - (128 << 23)) >> 14; int R, G, B; Y1 -= c->yuv2rgb_y_offset;