avfilter/vf_ssim360: fix integer overflow in tape_length allocation

tape_length * 8 overflows 32-bit int for large input widths. Then
av_malloc_array() allocates a tiny buffer while the subsequent
loop writes tape_length*8 BilinearMap entries, causing
heap-buffer-overflow.

Validate the value in float before converting to int and left
shifting, to avoid both float-to-int and signed left shift
overflow UB. Also split av_malloc_array() arguments to avoid
the multiplication overflow.

Fixes: #21511

Signed-off-by: Zhao Zhili <zhilizhao@tencent.com>
This commit is contained in:
Zhao Zhili
2026-03-23 16:21:24 +08:00
parent b796d72eb2
commit b62ae766c1

View File

@@ -1034,10 +1034,16 @@ generate_eye_tape_map(SSIM360Context *s,
float x_range = end_x - start_x;
// Ensure tape length is a multiple of 4, for full SSIM block coverage
int tape_length = s->tape_length[plane] = ((int)ROUNDED_DIV(x_range, 4)) << 2;
float tape_length_f = ROUNDED_DIV(x_range, 4);
int tape_length;
s->ref_tape_map[plane][eye] = av_malloc_array(tape_length * 8, sizeof(BilinearMap));
s->main_tape_map[plane][eye] = av_malloc_array(tape_length * 8, sizeof(BilinearMap));
if (!(tape_length_f > 0.f) || tape_length_f > INT_MAX / 4.0f)
return AVERROR(EINVAL);
tape_length = s->tape_length[plane] = (int)tape_length_f << 2;
s->ref_tape_map[plane][eye] = av_malloc_array(tape_length, 8 * sizeof(BilinearMap));
s->main_tape_map[plane][eye] = av_malloc_array(tape_length, 8 * sizeof(BilinearMap));
if (!s->ref_tape_map[plane][eye] || !s->main_tape_map[plane][eye])
return AVERROR(ENOMEM);