From 21d568be179c54a1596d1377b4da7fbe755bfe7f Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 5 Mar 2013 15:13:04 +0100 Subject: [PATCH 1/8] shorten: set invalid channels count to 0 Prevent the loop shorten_decode_close from writing and freeing out of the array boundary. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Luca Barbato CC: libav-stable@libav.org (cherry picked from commit c10da30d8426a1f681d99a780b6e311f7fb4e5c5) --- libavcodec/shorten.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c index 1dc010f441..50fc506440 100644 --- a/libavcodec/shorten.c +++ b/libavcodec/shorten.c @@ -345,6 +345,7 @@ static int read_header(ShortenContext *s) s->channels = get_uint(s, CHANSIZE); if (s->channels <= 0 || s->channels > MAX_CHANNELS) { av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels); + s->channels = 0; return -1; } s->avctx->channels = s->channels; From 97cc2f286f9e3eed1a00034367ebca58cc05ee39 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 5 Mar 2013 16:11:28 +0100 Subject: [PATCH 2/8] shorten: K&R formatting cosmetics (cherry picked from commit a2ad554def214d2d03b7c16f68dc081a8622f9ca) Signed-off-by: Luca Barbato --- libavcodec/shorten.c | 217 ++++++++++++++++++++++--------------------- 1 file changed, 112 insertions(+), 105 deletions(-) diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c index 50fc506440..f121a768ef 100644 --- a/libavcodec/shorten.c +++ b/libavcodec/shorten.c @@ -109,10 +109,10 @@ typedef struct ShortenContext { int got_quit_command; } ShortenContext; -static av_cold int shorten_decode_init(AVCodecContext * avctx) +static av_cold int shorten_decode_init(AVCodecContext *avctx) { ShortenContext *s = avctx->priv_data; - s->avctx = avctx; + s->avctx = avctx; avctx->sample_fmt = AV_SAMPLE_FMT_S16P; avcodec_get_frame_defaults(&s->frame); @@ -127,17 +127,20 @@ static int allocate_buffers(ShortenContext *s) int *coeffs; void *tmp_ptr; - for (chan=0; chanchannels; chan++) { - if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){ + for (chan = 0; chan < s->channels; chan++) { + if (FFMAX(1, s->nmean) >= UINT_MAX / sizeof(int32_t)) { av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n"); return -1; } - if(s->blocksize + s->nwrap >= UINT_MAX/sizeof(int32_t) || s->blocksize + s->nwrap <= (unsigned)s->nwrap){ - av_log(s->avctx, AV_LOG_ERROR, "s->blocksize + s->nwrap too large\n"); + if (s->blocksize + s->nwrap >= UINT_MAX / sizeof(int32_t) || + s->blocksize + s->nwrap <= (unsigned)s->nwrap) { + av_log(s->avctx, AV_LOG_ERROR, + "s->blocksize + s->nwrap too large\n"); return -1; } - tmp_ptr = av_realloc(s->offset[chan], sizeof(int32_t)*FFMAX(1, s->nmean)); + tmp_ptr = + av_realloc(s->offset[chan], sizeof(int32_t) * FFMAX(1, s->nmean)); if (!tmp_ptr) return AVERROR(ENOMEM); s->offset[chan] = tmp_ptr; @@ -147,7 +150,7 @@ static int allocate_buffers(ShortenContext *s) if (!tmp_ptr) return AVERROR(ENOMEM); s->decoded_base[chan] = tmp_ptr; - for (i=0; inwrap; i++) + for (i = 0; i < s->nwrap; i++) s->decoded_base[chan][i] = 0; s->decoded[chan] = s->decoded_base[chan] + s->nwrap; } @@ -160,7 +163,6 @@ static int allocate_buffers(ShortenContext *s) return 0; } - static inline unsigned int get_uint(ShortenContext *s, int k) { if (s->version != 0) @@ -168,7 +170,6 @@ static inline unsigned int get_uint(ShortenContext *s, int k) return get_ur_golomb_shorten(&s->gb, k); } - static void fix_bitshift(ShortenContext *s, int32_t *buffer) { int i; @@ -178,22 +179,20 @@ static void fix_bitshift(ShortenContext *s, int32_t *buffer) buffer[i] <<= s->bitshift; } - static int init_offset(ShortenContext *s) { int32_t mean = 0; - int chan, i; + int chan, i; int nblock = FFMAX(1, s->nmean); /* initialise offset */ - switch (s->internal_ftype) - { - case TYPE_S16HL: - case TYPE_S16LH: - mean = 0; - break; - default: - av_log(s->avctx, AV_LOG_ERROR, "unknown audio type"); - return AVERROR_INVALIDDATA; + switch (s->internal_ftype) { + case TYPE_S16HL: + case TYPE_S16LH: + mean = 0; + break; + default: + av_log(s->avctx, AV_LOG_ERROR, "unknown audio type"); + return AVERROR_INVALIDDATA; } for (chan = 0; chan < s->channels; chan++) @@ -208,21 +207,20 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header, int len; short wave_format; - - if (bytestream_get_le32(&header) != MKTAG('R','I','F','F')) { + if (bytestream_get_le32(&header) != MKTAG('R', 'I', 'F', 'F')) { av_log(avctx, AV_LOG_ERROR, "missing RIFF tag\n"); return -1; } - header += 4; /* chunk size */; + header += 4; /* chunk size */ - if (bytestream_get_le32(&header) != MKTAG('W','A','V','E')) { + if (bytestream_get_le32(&header) != MKTAG('W', 'A', 'V', 'E')) { av_log(avctx, AV_LOG_ERROR, "missing WAVE tag\n"); return -1; } - while (bytestream_get_le32(&header) != MKTAG('f','m','t',' ')) { - len = bytestream_get_le32(&header); + while (bytestream_get_le32(&header) != MKTAG('f', 'm', 't', ' ')) { + len = bytestream_get_le32(&header); header += len; } len = bytestream_get_le32(&header); @@ -235,11 +233,11 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header, wave_format = bytestream_get_le16(&header); switch (wave_format) { - case WAVE_FORMAT_PCM: - break; - default: - av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n"); - return -1; + case WAVE_FORMAT_PCM: + break; + default: + av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n"); + return -1; } header += 2; // skip channels (already got from shorten header) @@ -288,11 +286,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel, /* read/validate prediction order */ pred_order = get_ur_golomb_shorten(&s->gb, LPCQSIZE); if (pred_order > s->nwrap) { - av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n", pred_order); + av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n", + pred_order); return AVERROR(EINVAL); } /* read LPC coefficients */ - for (i=0; icoeffs[i] = get_sr_golomb_shorten(&s->gb, LPCQUANT); coeffs = s->coeffs; @@ -300,7 +299,7 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel, } else { /* fixed LPC coeffs */ pred_order = command; - coeffs = fixed_coeffs[pred_order-1]; + coeffs = fixed_coeffs[pred_order - 1]; qshift = 0; } @@ -311,11 +310,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel, /* decode residual and do LPC prediction */ init_sum = pred_order ? (command == FN_QLPC ? s->lpcqoffset : 0) : coffset; - for (i=0; i < s->blocksize; i++) { + for (i = 0; i < s->blocksize; i++) { sum = init_sum; - for (j=0; jdecoded[channel][i-j-1]; - s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + (sum >> qshift); + for (j = 0; j < pred_order; j++) + sum += coeffs[j] * s->decoded[channel][i - j - 1]; + s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + + (sum >> qshift); } /* add offset to current samples */ @@ -336,10 +336,10 @@ static int read_header(ShortenContext *s) return -1; } - s->lpcqoffset = 0; - s->blocksize = DEFAULT_BLOCK_SIZE; - s->nmean = -1; - s->version = get_bits(&s->gb, 8); + s->lpcqoffset = 0; + s->blocksize = DEFAULT_BLOCK_SIZE; + s->nmean = -1; + s->version = get_bits(&s->gb, 8); s->internal_ftype = get_uint(s, TYPESIZE); s->channels = get_uint(s, CHANSIZE); @@ -356,19 +356,19 @@ static int read_header(ShortenContext *s) blocksize = get_uint(s, av_log2(DEFAULT_BLOCK_SIZE)); if (!blocksize || blocksize > MAX_BLOCKSIZE) { - av_log(s->avctx, AV_LOG_ERROR, "invalid or unsupported block size: %d\n", + av_log(s->avctx, AV_LOG_ERROR, + "invalid or unsupported block size: %d\n", blocksize); return AVERROR(EINVAL); } s->blocksize = blocksize; - maxnlpc = get_uint(s, LPCQSIZE); + maxnlpc = get_uint(s, LPCQSIZE); s->nmean = get_uint(s, 0); skip_bytes = get_uint(s, NSKIPSIZE); - for (i=0; igb, 8); - } } s->nwrap = FFMAX(NWRAP, maxnlpc); @@ -382,17 +382,20 @@ static int read_header(ShortenContext *s) s->lpcqoffset = V2LPCQOFFSET; if (get_ur_golomb_shorten(&s->gb, FNSIZE) != FN_VERBATIM) { - av_log(s->avctx, AV_LOG_ERROR, "missing verbatim section at beginning of stream\n"); + av_log(s->avctx, AV_LOG_ERROR, + "missing verbatim section at beginning of stream\n"); return -1; } s->header_size = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE); - if (s->header_size >= OUT_BUFFER_SIZE || s->header_size < CANONICAL_HEADER_SIZE) { - av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n", s->header_size); + if (s->header_size >= OUT_BUFFER_SIZE || + s->header_size < CANONICAL_HEADER_SIZE) { + av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n", + s->header_size); return -1; } - for (i=0; iheader_size; i++) + for (i = 0; i < s->header_size; i++) s->header[i] = (char)get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE); if (decode_wave_header(s->avctx, s->header, s->header_size) < 0) @@ -410,15 +413,15 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, int *got_frame_ptr, AVPacket *avpkt) { const uint8_t *buf = avpkt->data; - int buf_size = avpkt->size; - ShortenContext *s = avctx->priv_data; + int buf_size = avpkt->size; + ShortenContext *s = avctx->priv_data; int i, input_buf_size = 0; int ret; /* allocate internal bitstream buffer */ - if(s->max_framesize == 0){ + if (s->max_framesize == 0) { void *tmp_ptr; - s->max_framesize= 1024; // should hopefully be enough for the first header + s->max_framesize = 1024; // should hopefully be enough for the first header tmp_ptr = av_fast_realloc(s->bitstream, &s->allocated_bitstream_size, s->max_framesize); if (!tmp_ptr) { @@ -429,29 +432,32 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, } /* append current packet data to bitstream buffer */ - if(1 && s->max_framesize){//FIXME truncated - buf_size= FFMIN(buf_size, s->max_framesize - s->bitstream_size); - input_buf_size= buf_size; + if (1 && s->max_framesize) { //FIXME truncated + buf_size = FFMIN(buf_size, s->max_framesize - s->bitstream_size); + input_buf_size = buf_size; - if(s->bitstream_index + s->bitstream_size + buf_size > s->allocated_bitstream_size){ - memmove(s->bitstream, &s->bitstream[s->bitstream_index], s->bitstream_size); - s->bitstream_index=0; + if (s->bitstream_index + s->bitstream_size + buf_size > + s->allocated_bitstream_size) { + memmove(s->bitstream, &s->bitstream[s->bitstream_index], + s->bitstream_size); + s->bitstream_index = 0; } if (buf) - memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf, buf_size); - buf= &s->bitstream[s->bitstream_index]; - buf_size += s->bitstream_size; - s->bitstream_size= buf_size; + memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf, + buf_size); + buf = &s->bitstream[s->bitstream_index]; + buf_size += s->bitstream_size; + s->bitstream_size = buf_size; /* do not decode until buffer has at least max_framesize bytes or - the end of the file has been reached */ + * the end of the file has been reached */ if (buf_size < s->max_framesize && avpkt->data) { *got_frame_ptr = 0; return input_buf_size; } } /* init and position bitstream reader */ - init_get_bits(&s->gb, buf, buf_size*8); + init_get_bits(&s->gb, buf, buf_size * 8); skip_bits(&s->gb, s->bitindex); /* process header or next subblock */ @@ -473,7 +479,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, int cmd; int len; - if (get_bits_left(&s->gb) < 3+FNSIZE) { + if (get_bits_left(&s->gb) < 3 + FNSIZE) { *got_frame_ptr = 0; break; } @@ -489,32 +495,32 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, if (!is_audio_command[cmd]) { /* process non-audio command */ switch (cmd) { - case FN_VERBATIM: - len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE); - while (len--) { - get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE); - } - break; - case FN_BITSHIFT: - s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE); - break; - case FN_BLOCKSIZE: { - int blocksize = get_uint(s, av_log2(s->blocksize)); - if (blocksize > s->blocksize) { - av_log(avctx, AV_LOG_ERROR, "Increasing block size is not supported\n"); - return AVERROR_PATCHWELCOME; - } - if (!blocksize || blocksize > MAX_BLOCKSIZE) { - av_log(avctx, AV_LOG_ERROR, "invalid or unsupported " - "block size: %d\n", blocksize); - return AVERROR(EINVAL); - } - s->blocksize = blocksize; - break; + case FN_VERBATIM: + len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE); + while (len--) + get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE); + break; + case FN_BITSHIFT: + s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE); + break; + case FN_BLOCKSIZE: { + int blocksize = get_uint(s, av_log2(s->blocksize)); + if (blocksize > s->blocksize) { + av_log(avctx, AV_LOG_ERROR, + "Increasing block size is not supported\n"); + return AVERROR_PATCHWELCOME; } - case FN_QUIT: - s->got_quit_command = 1; - break; + if (!blocksize || blocksize > MAX_BLOCKSIZE) { + av_log(avctx, AV_LOG_ERROR, "invalid or unsupported " + "block size: %d\n", blocksize); + return AVERROR(EINVAL); + } + s->blocksize = blocksize; + break; + } + case FN_QUIT: + s->got_quit_command = 1; + break; } if (cmd == FN_BLOCKSIZE || cmd == FN_QUIT) { *got_frame_ptr = 0; @@ -540,7 +546,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, coffset = s->offset[channel][0]; else { int32_t sum = (s->version < 2) ? 0 : s->nmean / 2; - for (i=0; inmean; i++) + for (i = 0; i < s->nmean; i++) sum += s->offset[channel][i]; coffset = sum / s->nmean; if (s->version >= 2) @@ -549,21 +555,22 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, /* decode samples for this channel */ if (cmd == FN_ZERO) { - for (i=0; iblocksize; i++) + for (i = 0; i < s->blocksize; i++) s->decoded[channel][i] = 0; } else { - if ((ret = decode_subframe_lpc(s, cmd, channel, residual_size, coffset)) < 0) + if ((ret = decode_subframe_lpc(s, cmd, channel, + residual_size, coffset)) < 0) return ret; } /* update means with info from the current block */ if (s->nmean > 0) { int32_t sum = (s->version < 2) ? 0 : s->blocksize / 2; - for (i=0; iblocksize; i++) + for (i = 0; i < s->blocksize; i++) sum += s->decoded[channel][i]; - for (i=1; inmean; i++) - s->offset[channel][i-1] = s->offset[channel][i]; + for (i = 1; i < s->nmean; i++) + s->offset[channel][i - 1] = s->offset[channel][i]; if (s->version < 2) s->offset[channel][s->nmean - 1] = sum / s->blocksize; @@ -572,11 +579,11 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, } /* copy wrap samples for use with next block */ - for (i=-s->nwrap; i<0; i++) + for (i = -s->nwrap; i < 0; i++) s->decoded[channel][i] = s->decoded[channel][i + s->blocksize]; /* shift samples to add in unused zero bits which were removed - during encoding */ + * during encoding */ fix_bitshift(s, s->decoded[channel]); /* if this is the last channel in the block, output the samples */ @@ -601,12 +608,12 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, *got_frame_ptr = 0; finish_frame: - s->bitindex = get_bits_count(&s->gb) - 8*((get_bits_count(&s->gb))/8); - i= (get_bits_count(&s->gb))/8; + s->bitindex = get_bits_count(&s->gb) - 8 * (get_bits_count(&s->gb) / 8); + i = get_bits_count(&s->gb) / 8; if (i > buf_size) { av_log(s->avctx, AV_LOG_ERROR, "overread: %d\n", i - buf_size); - s->bitstream_size=0; - s->bitstream_index=0; + s->bitstream_size = 0; + s->bitstream_index = 0; return -1; } if (s->bitstream_size) { From 0daf1428e82926dc5a8c72a0ff4c93aaa8a84ed9 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 5 Mar 2013 16:34:16 +0100 Subject: [PATCH 3/8] shorten: report meaningful errors (cherry picked from commit 4c364eb2b856fc33cf7b42f7c7b979e69fde5f3a) Signed-off-by: Luca Barbato --- libavcodec/shorten.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c index f121a768ef..89346b4aad 100644 --- a/libavcodec/shorten.c +++ b/libavcodec/shorten.c @@ -130,13 +130,13 @@ static int allocate_buffers(ShortenContext *s) for (chan = 0; chan < s->channels; chan++) { if (FFMAX(1, s->nmean) >= UINT_MAX / sizeof(int32_t)) { av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n"); - return -1; + return AVERROR_INVALIDDATA; } if (s->blocksize + s->nwrap >= UINT_MAX / sizeof(int32_t) || s->blocksize + s->nwrap <= (unsigned)s->nwrap) { av_log(s->avctx, AV_LOG_ERROR, "s->blocksize + s->nwrap too large\n"); - return -1; + return AVERROR_INVALIDDATA; } tmp_ptr = @@ -209,14 +209,14 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header, if (bytestream_get_le32(&header) != MKTAG('R', 'I', 'F', 'F')) { av_log(avctx, AV_LOG_ERROR, "missing RIFF tag\n"); - return -1; + return AVERROR_INVALIDDATA; } header += 4; /* chunk size */ if (bytestream_get_le32(&header) != MKTAG('W', 'A', 'V', 'E')) { av_log(avctx, AV_LOG_ERROR, "missing WAVE tag\n"); - return -1; + return AVERROR_INVALIDDATA; } while (bytestream_get_le32(&header) != MKTAG('f', 'm', 't', ' ')) { @@ -227,7 +227,7 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header, if (len < 16) { av_log(avctx, AV_LOG_ERROR, "fmt chunk was too short\n"); - return -1; + return AVERROR_INVALIDDATA; } wave_format = bytestream_get_le16(&header); @@ -237,7 +237,7 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header, break; default: av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n"); - return -1; + return AVERROR(ENOSYS); } header += 2; // skip channels (already got from shorten header) @@ -248,7 +248,7 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header, if (avctx->bits_per_coded_sample != 16) { av_log(avctx, AV_LOG_ERROR, "unsupported number of bits per sample\n"); - return -1; + return AVERROR(ENOSYS); } len -= 16; @@ -333,7 +333,7 @@ static int read_header(ShortenContext *s) /* shorten signature */ if (get_bits_long(&s->gb, 32) != AV_RB32("ajkg")) { av_log(s->avctx, AV_LOG_ERROR, "missing shorten magic 'ajkg'\n"); - return -1; + return AVERROR_INVALIDDATA; } s->lpcqoffset = 0; @@ -346,7 +346,7 @@ static int read_header(ShortenContext *s) if (s->channels <= 0 || s->channels > MAX_CHANNELS) { av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels); s->channels = 0; - return -1; + return AVERROR_INVALIDDATA; } s->avctx->channels = s->channels; @@ -384,7 +384,7 @@ static int read_header(ShortenContext *s) if (get_ur_golomb_shorten(&s->gb, FNSIZE) != FN_VERBATIM) { av_log(s->avctx, AV_LOG_ERROR, "missing verbatim section at beginning of stream\n"); - return -1; + return AVERROR_INVALIDDATA; } s->header_size = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE); @@ -392,14 +392,14 @@ static int read_header(ShortenContext *s) s->header_size < CANONICAL_HEADER_SIZE) { av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n", s->header_size); - return -1; + return AVERROR_INVALIDDATA; } for (i = 0; i < s->header_size; i++) s->header[i] = (char)get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE); - if (decode_wave_header(s->avctx, s->header, s->header_size) < 0) - return -1; + if ((ret = decode_wave_header(s->avctx, s->header, s->header_size)) < 0) + return ret; s->cur_chan = 0; s->bitshift = 0; @@ -614,7 +614,7 @@ finish_frame: av_log(s->avctx, AV_LOG_ERROR, "overread: %d\n", i - buf_size); s->bitstream_size = 0; s->bitstream_index = 0; - return -1; + return AVERROR_INVALIDDATA; } if (s->bitstream_size) { s->bitstream_index += i; From 88089eecfd7e604d40d078b4f4206c647cb2e2b4 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 5 Mar 2013 17:12:35 +0100 Subject: [PATCH 4/8] shorten: use the unsigned type where needed get_uint returns an unsigned value, use an unsigned to store blocksize to make sure the comparison logic is correct and report correctly the error for the channel count not supported. CC: libav-stable@libav.org (cherry picked from commit 5cf7c72757779a740e897a97710aac044fe5258c) Signed-off-by: Luca Barbato --- libavcodec/shorten.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c index 89346b4aad..0b4a473892 100644 --- a/libavcodec/shorten.c +++ b/libavcodec/shorten.c @@ -84,7 +84,7 @@ typedef struct ShortenContext { GetBitContext gb; int min_framesize, max_framesize; - int channels; + unsigned channels; int32_t *decoded[MAX_CHANNELS]; int32_t *decoded_base[MAX_CHANNELS]; @@ -343,7 +343,11 @@ static int read_header(ShortenContext *s) s->internal_ftype = get_uint(s, TYPESIZE); s->channels = get_uint(s, CHANSIZE); - if (s->channels <= 0 || s->channels > MAX_CHANNELS) { + if (!s->channels) { + av_log(s->avctx, AV_LOG_ERROR, "No channels reported\n"); + return AVERROR_INVALIDDATA; + } + if (s->channels > MAX_CHANNELS) { av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels); s->channels = 0; return AVERROR_INVALIDDATA; @@ -352,7 +356,8 @@ static int read_header(ShortenContext *s) /* get blocksize if version > 0 */ if (s->version > 0) { - int skip_bytes, blocksize; + int skip_bytes; + unsigned blocksize; blocksize = get_uint(s, av_log2(DEFAULT_BLOCK_SIZE)); if (!blocksize || blocksize > MAX_BLOCKSIZE) { @@ -504,7 +509,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data, s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE); break; case FN_BLOCKSIZE: { - int blocksize = get_uint(s, av_log2(s->blocksize)); + unsigned blocksize = get_uint(s, av_log2(s->blocksize)); if (blocksize > s->blocksize) { av_log(avctx, AV_LOG_ERROR, "Increasing block size is not supported\n"); From 0b0e87bb544be1926c4ee86c25789d5d6c3c6255 Mon Sep 17 00:00:00 2001 From: Xi Wang Date: Fri, 15 Mar 2013 06:31:21 -0400 Subject: [PATCH 5/8] atrac3: avoid oversized shifting in decode_bytes() When `off' is 0, `0x537F6103 << 32' in the following expression invokes undefined behavior, the result of which is not necessarily 0. (0x537F6103 >> (off * 8)) | (0x537F6103 << (32 - (off * 8))) Avoid oversized shifting. CC: libav-stable@libav.org Signed-off-by: Xi Wang Signed-off-by: Luca Barbato (cherry picked from commit eba1ff31304e407db3cefd7532108408f364367b) --- libavcodec/atrac3.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libavcodec/atrac3.c b/libavcodec/atrac3.c index a46b0b1277..910c15e473 100644 --- a/libavcodec/atrac3.c +++ b/libavcodec/atrac3.c @@ -164,7 +164,10 @@ static int decode_bytes(const uint8_t *input, uint8_t *out, int bytes) off = (intptr_t)input & 3; buf = (const uint32_t *)(input - off); - c = av_be2ne32((0x537F6103 >> (off * 8)) | (0x537F6103 << (32 - (off * 8)))); + if (off) + c = av_be2ne32((0x537F6103U >> (off * 8)) | (0x537F6103U << (32 - (off * 8)))); + else + c = av_be2ne32(0x537F6103U); bytes += 3 + off; for (i = 0; i < bytes / 4; i++) output[i] = c ^ buf[i]; From 9d4355d90a6a8fc49f6ff05f98b86e37c262ac91 Mon Sep 17 00:00:00 2001 From: Xi Wang Date: Fri, 15 Mar 2013 07:11:47 -0400 Subject: [PATCH 6/8] flacdec: simplify bounds checking in flac_probe() Simplify `p->buf > p->buf + p->buf_size - 4' as `p->buf_size < 4'. Avoid a possible out-of-bounds pointer, which is undefined behavior in C. CC: libav-stable@libav.org Signed-off-by: Xi Wang Signed-off-by: Luca Barbato (cherry picked from commit 8425d693eefbedbb41f91735614d41067695aa37) --- libavformat/flacdec.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libavformat/flacdec.c b/libavformat/flacdec.c index f19f95d901..fcacf190bc 100644 --- a/libavformat/flacdec.c +++ b/libavformat/flacdec.c @@ -278,11 +278,9 @@ static int flac_read_header(AVFormatContext *s) static int flac_probe(AVProbeData *p) { - uint8_t *bufptr = p->buf; - uint8_t *end = p->buf + p->buf_size; - - if(bufptr > end-4 || memcmp(bufptr, "fLaC", 4)) return 0; - else return AVPROBE_SCORE_MAX/2; + if (p->buf_size < 4 || memcmp(p->buf, "fLaC", 4)) + return 0; + return AVPROBE_SCORE_MAX/2; } AVInputFormat ff_flac_demuxer = { From 22c27e1f4a6ff3d8d0409b30c8a9edb689ffdeaa Mon Sep 17 00:00:00 2001 From: Xi Wang Date: Fri, 15 Mar 2013 06:59:22 -0400 Subject: [PATCH 7/8] lzo: fix overflow checking in copy_backptr() The check `src > dst' in the form `&c->out[-back] > c->out' invokes pointer overflow, which is undefined behavior in C. Remove the check. Also replace `&c->out[-back] < c->out_start' with a safe form `c->out - c->out_start < back' to avoid overflow. CC: libav-stable@libav.org Signed-off-by: Xi Wang Signed-off-by: Luca Barbato (cherry picked from commit ca6c3f2c53be70aa3c38e8f1292809db89ea1ba6) --- libavutil/lzo.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libavutil/lzo.c b/libavutil/lzo.c index eff3cd2333..5c5ebc850a 100644 --- a/libavutil/lzo.c +++ b/libavutil/lzo.c @@ -110,9 +110,8 @@ static inline void copy(LZOContext *c, int cnt) */ static inline void copy_backptr(LZOContext *c, int back, int cnt) { - register const uint8_t *src = &c->out[-back]; register uint8_t *dst = c->out; - if (src < c->out_start || src > dst) { + if (dst - c->out_start < back) { c->error |= AV_LZO_INVALID_BACKPTR; return; } From c50241080d7599c90fc8b4e74c5f8d62a4caae52 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Thu, 28 Feb 2013 08:47:21 +0100 Subject: [PATCH 8/8] vf_hqdn3d: fix uninitialized variable use CC:libav-stable@libav.org (cherry picked from commit d0a863ac891eae49ceaa4de7f759270bc87e668d) Conflicts: libavfilter/vf_hqdn3d.c --- libavfilter/vf_hqdn3d.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavfilter/vf_hqdn3d.c b/libavfilter/vf_hqdn3d.c index 085900fbce..30f7817ec0 100644 --- a/libavfilter/vf_hqdn3d.c +++ b/libavfilter/vf_hqdn3d.c @@ -333,6 +333,7 @@ static int filter_frame(AVFilterLink *inlink, AVFilterBufferRef *in) direct = 1; out = in; } else { + direct = 0; out = ff_get_video_buffer(outlink, AV_PERM_WRITE, outlink->w, outlink->h); if (!out) { avfilter_unref_bufferp(&in);