From 7466d8a8509bcb2f54e4f9e149bded8f413b6e46 Mon Sep 17 00:00:00 2001 From: Ruikai Peng Date: Tue, 31 Mar 2026 21:59:38 -0400 Subject: [PATCH] avformat/whip: check RTP history packet size before RTX retransmission handle_rtx_packet() constructs an RTX packet by shifting the payload of a history entry to insert the original sequence number. It uses memmove with length (ori_size - 12), but never checks that ori_size is at least 12 bytes (the minimum RTP header size). Zero-initialized history slots have seq == 0 and size == 0. rtp_history_find() only compares sequence numbers, so an RTCP NACK requesting seq 0 early in a session matches such a slot. The subtraction then wraps to a huge value when converted to size_t, causing a stack buffer overflow in memmove(). Add a little size check to reject history entries smaller than and valid RTP header before any arithmetic on their size. Found-by: Pwno --- libavformat/whip.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavformat/whip.c b/libavformat/whip.c index 0d39247a03..c4a55192a8 100644 --- a/libavformat/whip.c +++ b/libavformat/whip.c @@ -1916,6 +1916,12 @@ static void handle_rtx_packet(AVFormatContext *s, uint16_t seq) ori_buf = it->buf; ori_size = it->size; + /* A valid RTP packet must have at least a RTP header. */ + if (ori_size < WHIP_RTP_HEADER_SIZE) { + av_log(whip, AV_LOG_WARNING, "RTX history packet too small, size=%d\n", ori_size); + goto end; + } + /* RTX packet format: header + original seq (2 bytes) + payload */ if (ori_size + 2 > sizeof(rtx_buf)) { av_log(whip, AV_LOG_WARNING, "RTX packet is too large, size=%d\n", ori_size);