Merge commit 'c50241080d7599c90fc8b4e74c5f8d62a4caae52' into release/1.1

* commit 'c50241080d7599c90fc8b4e74c5f8d62a4caae52':
  vf_hqdn3d: fix uninitialized variable use
  lzo: fix overflow checking in copy_backptr()
  flacdec: simplify bounds checking in flac_probe()
  atrac3: avoid oversized shifting in decode_bytes()
  shorten: use the unsigned type where needed
  shorten: report meaningful errors
  shorten: K&R formatting cosmetics
  shorten: set invalid channels count to 0

Conflicts:
	libavcodec/shorten.c
	libavformat/flacdec.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer
2013-03-18 02:54:35 +01:00
5 changed files with 136 additions and 122 deletions

View File

@@ -165,7 +165,10 @@ static int decode_bytes(const uint8_t *input, uint8_t *out, int bytes)
off = (intptr_t)input & 3;
buf = (const uint32_t *)(input - off);
c = av_be2ne32((0x537F6103 >> (off * 8)) | (0x537F6103 << (32 - (off * 8))));
if (off)
c = av_be2ne32((0x537F6103U >> (off * 8)) | (0x537F6103U << (32 - (off * 8))));
else
c = av_be2ne32(0x537F6103U);
bytes += 3 + off;
for (i = 0; i < bytes / 4; i++)
output[i] = c ^ buf[i];

View File

@@ -88,7 +88,7 @@ typedef struct ShortenContext {
GetBitContext gb;
int min_framesize, max_framesize;
int channels;
unsigned channels;
int32_t *decoded[MAX_CHANNELS];
int32_t *decoded_base[MAX_CHANNELS];
@@ -113,10 +113,10 @@ typedef struct ShortenContext {
int got_quit_command;
} ShortenContext;
static av_cold int shorten_decode_init(AVCodecContext * avctx)
static av_cold int shorten_decode_init(AVCodecContext *avctx)
{
ShortenContext *s = avctx->priv_data;
s->avctx = avctx;
s->avctx = avctx;
avcodec_get_frame_defaults(&s->frame);
avctx->coded_frame = &s->frame;
@@ -130,17 +130,20 @@ static int allocate_buffers(ShortenContext *s)
int *coeffs;
void *tmp_ptr;
for (chan=0; chan<s->channels; chan++) {
if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){
for (chan = 0; chan < s->channels; chan++) {
if (FFMAX(1, s->nmean) >= UINT_MAX / sizeof(int32_t)) {
av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n");
return AVERROR_INVALIDDATA;
}
if(s->blocksize + s->nwrap >= UINT_MAX/sizeof(int32_t) || s->blocksize + s->nwrap <= (unsigned)s->nwrap){
av_log(s->avctx, AV_LOG_ERROR, "s->blocksize + s->nwrap too large\n");
if (s->blocksize + s->nwrap >= UINT_MAX / sizeof(int32_t) ||
s->blocksize + s->nwrap <= (unsigned)s->nwrap) {
av_log(s->avctx, AV_LOG_ERROR,
"s->blocksize + s->nwrap too large\n");
return AVERROR_INVALIDDATA;
}
tmp_ptr = av_realloc(s->offset[chan], sizeof(int32_t)*FFMAX(1, s->nmean));
tmp_ptr =
av_realloc(s->offset[chan], sizeof(int32_t) * FFMAX(1, s->nmean));
if (!tmp_ptr)
return AVERROR(ENOMEM);
s->offset[chan] = tmp_ptr;
@@ -150,7 +153,7 @@ static int allocate_buffers(ShortenContext *s)
if (!tmp_ptr)
return AVERROR(ENOMEM);
s->decoded_base[chan] = tmp_ptr;
for (i=0; i<s->nwrap; i++)
for (i = 0; i < s->nwrap; i++)
s->decoded_base[chan][i] = 0;
s->decoded[chan] = s->decoded_base[chan] + s->nwrap;
}
@@ -163,7 +166,6 @@ static int allocate_buffers(ShortenContext *s)
return 0;
}
static inline unsigned int get_uint(ShortenContext *s, int k)
{
if (s->version != 0)
@@ -171,7 +173,6 @@ static inline unsigned int get_uint(ShortenContext *s, int k)
return get_ur_golomb_shorten(&s->gb, k);
}
static void fix_bitshift(ShortenContext *s, int32_t *buffer)
{
int i;
@@ -181,26 +182,24 @@ static void fix_bitshift(ShortenContext *s, int32_t *buffer)
buffer[i] <<= s->bitshift;
}
static int init_offset(ShortenContext *s)
{
int32_t mean = 0;
int chan, i;
int chan, i;
int nblock = FFMAX(1, s->nmean);
/* initialise offset */
switch (s->internal_ftype)
{
case TYPE_U8:
s->avctx->sample_fmt = AV_SAMPLE_FMT_U8P;
mean = 0x80;
break;
case TYPE_S16HL:
case TYPE_S16LH:
s->avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
break;
default:
av_log(s->avctx, AV_LOG_ERROR, "unknown audio type\n");
return AVERROR_PATCHWELCOME;
switch (s->internal_ftype) {
case TYPE_U8:
s->avctx->sample_fmt = AV_SAMPLE_FMT_U8P;
mean = 0x80;
break;
case TYPE_S16HL:
case TYPE_S16LH:
s->avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
break;
default:
av_log(s->avctx, AV_LOG_ERROR, "unknown audio type\n");
return AVERROR_PATCHWELCOME;
}
for (chan = 0; chan < s->channels; chan++)
@@ -216,21 +215,21 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
short wave_format;
const uint8_t *end= header + header_size;
if (bytestream_get_le32(&header) != MKTAG('R','I','F','F')) {
if (bytestream_get_le32(&header) != MKTAG('R', 'I', 'F', 'F')) {
av_log(avctx, AV_LOG_ERROR, "missing RIFF tag\n");
return AVERROR_INVALIDDATA;
}
header += 4; /* chunk size */;
header += 4; /* chunk size */
if (bytestream_get_le32(&header) != MKTAG('W','A','V','E')) {
if (bytestream_get_le32(&header) != MKTAG('W', 'A', 'V', 'E')) {
av_log(avctx, AV_LOG_ERROR, "missing WAVE tag\n");
return AVERROR_INVALIDDATA;
}
while (bytestream_get_le32(&header) != MKTAG('f','m','t',' ')) {
len = bytestream_get_le32(&header);
if(len<0 || end - header - 8 < len)
while (bytestream_get_le32(&header) != MKTAG('f', 'm', 't', ' ')) {
len = bytestream_get_le32(&header);
if (len<0 || end - header - 8 < len)
return AVERROR_INVALIDDATA;
header += len;
}
@@ -244,11 +243,11 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
wave_format = bytestream_get_le16(&header);
switch (wave_format) {
case WAVE_FORMAT_PCM:
break;
default:
av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n");
return AVERROR_PATCHWELCOME;
case WAVE_FORMAT_PCM:
break;
default:
av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n");
return AVERROR(ENOSYS);
}
header += 2; // skip channels (already got from shorten header)
@@ -260,7 +259,7 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
if (bps != 16 && bps != 8) {
av_log(avctx, AV_LOG_ERROR, "unsupported number of bits per sample: %d\n", bps);
return AVERROR_INVALIDDATA;
return AVERROR(ENOSYS);
}
len -= 16;
@@ -286,11 +285,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
/* read/validate prediction order */
pred_order = get_ur_golomb_shorten(&s->gb, LPCQSIZE);
if (pred_order > s->nwrap) {
av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n", pred_order);
av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n",
pred_order);
return AVERROR(EINVAL);
}
/* read LPC coefficients */
for (i=0; i<pred_order; i++)
for (i = 0; i < pred_order; i++)
s->coeffs[i] = get_sr_golomb_shorten(&s->gb, LPCQUANT);
coeffs = s->coeffs;
@@ -298,7 +298,7 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
} else {
/* fixed LPC coeffs */
pred_order = command;
coeffs = fixed_coeffs[pred_order-1];
coeffs = fixed_coeffs[pred_order - 1];
qshift = 0;
}
@@ -309,11 +309,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
/* decode residual and do LPC prediction */
init_sum = pred_order ? (command == FN_QLPC ? s->lpcqoffset : 0) : coffset;
for (i=0; i < s->blocksize; i++) {
for (i = 0; i < s->blocksize; i++) {
sum = init_sum;
for (j=0; j<pred_order; j++)
sum += coeffs[j] * s->decoded[channel][i-j-1];
s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + (sum >> qshift);
for (j = 0; j < pred_order; j++)
sum += coeffs[j] * s->decoded[channel][i - j - 1];
s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) +
(sum >> qshift);
}
/* add offset to current samples */
@@ -334,14 +335,18 @@ static int read_header(ShortenContext *s)
return AVERROR_INVALIDDATA;
}
s->lpcqoffset = 0;
s->blocksize = DEFAULT_BLOCK_SIZE;
s->nmean = -1;
s->version = get_bits(&s->gb, 8);
s->lpcqoffset = 0;
s->blocksize = DEFAULT_BLOCK_SIZE;
s->nmean = -1;
s->version = get_bits(&s->gb, 8);
s->internal_ftype = get_uint(s, TYPESIZE);
s->channels = get_uint(s, CHANSIZE);
if (s->channels <= 0 || s->channels > MAX_CHANNELS) {
if (!s->channels) {
av_log(s->avctx, AV_LOG_ERROR, "No channels reported\n");
return AVERROR_INVALIDDATA;
}
if (s->channels > MAX_CHANNELS) {
av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels);
s->channels = 0;
return AVERROR_INVALIDDATA;
@@ -350,23 +355,24 @@ static int read_header(ShortenContext *s)
/* get blocksize if version > 0 */
if (s->version > 0) {
int skip_bytes, blocksize;
int skip_bytes;
unsigned blocksize;
blocksize = get_uint(s, av_log2(DEFAULT_BLOCK_SIZE));
if (!blocksize || blocksize > MAX_BLOCKSIZE) {
av_log(s->avctx, AV_LOG_ERROR, "invalid or unsupported block size: %d\n",
av_log(s->avctx, AV_LOG_ERROR,
"invalid or unsupported block size: %d\n",
blocksize);
return AVERROR(EINVAL);
}
s->blocksize = blocksize;
maxnlpc = get_uint(s, LPCQSIZE);
maxnlpc = get_uint(s, LPCQSIZE);
s->nmean = get_uint(s, 0);
skip_bytes = get_uint(s, NSKIPSIZE);
for (i=0; i<skip_bytes; i++) {
for (i = 0; i < skip_bytes; i++)
skip_bits(&s->gb, 8);
}
}
s->nwrap = FFMAX(NWRAP, maxnlpc);
@@ -380,17 +386,20 @@ static int read_header(ShortenContext *s)
s->lpcqoffset = V2LPCQOFFSET;
if (get_ur_golomb_shorten(&s->gb, FNSIZE) != FN_VERBATIM) {
av_log(s->avctx, AV_LOG_ERROR, "missing verbatim section at beginning of stream\n");
av_log(s->avctx, AV_LOG_ERROR,
"missing verbatim section at beginning of stream\n");
return AVERROR_INVALIDDATA;
}
s->header_size = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
if (s->header_size >= OUT_BUFFER_SIZE || s->header_size < CANONICAL_HEADER_SIZE) {
av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n", s->header_size);
if (s->header_size >= OUT_BUFFER_SIZE ||
s->header_size < CANONICAL_HEADER_SIZE) {
av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n",
s->header_size);
return AVERROR_INVALIDDATA;
}
for (i=0; i<s->header_size; i++)
for (i = 0; i < s->header_size; i++)
s->header[i] = (char)get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
if ((ret = decode_wave_header(s->avctx, s->header, s->header_size)) < 0)
@@ -408,15 +417,15 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
int *got_frame_ptr, AVPacket *avpkt)
{
const uint8_t *buf = avpkt->data;
int buf_size = avpkt->size;
ShortenContext *s = avctx->priv_data;
int buf_size = avpkt->size;
ShortenContext *s = avctx->priv_data;
int i, input_buf_size = 0;
int ret;
/* allocate internal bitstream buffer */
if(s->max_framesize == 0){
if (s->max_framesize == 0) {
void *tmp_ptr;
s->max_framesize= 8192; // should hopefully be enough for the first header
s->max_framesize = 8192; // should hopefully be enough for the first header
tmp_ptr = av_fast_realloc(s->bitstream, &s->allocated_bitstream_size,
s->max_framesize);
if (!tmp_ptr) {
@@ -427,29 +436,32 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
}
/* append current packet data to bitstream buffer */
if(1 && s->max_framesize){//FIXME truncated
buf_size= FFMIN(buf_size, s->max_framesize - s->bitstream_size);
input_buf_size= buf_size;
if (1 && s->max_framesize) { //FIXME truncated
buf_size = FFMIN(buf_size, s->max_framesize - s->bitstream_size);
input_buf_size = buf_size;
if(s->bitstream_index + s->bitstream_size + buf_size > s->allocated_bitstream_size){
memmove(s->bitstream, &s->bitstream[s->bitstream_index], s->bitstream_size);
s->bitstream_index=0;
if (s->bitstream_index + s->bitstream_size + buf_size >
s->allocated_bitstream_size) {
memmove(s->bitstream, &s->bitstream[s->bitstream_index],
s->bitstream_size);
s->bitstream_index = 0;
}
if (buf)
memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf, buf_size);
buf= &s->bitstream[s->bitstream_index];
buf_size += s->bitstream_size;
s->bitstream_size= buf_size;
memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf,
buf_size);
buf = &s->bitstream[s->bitstream_index];
buf_size += s->bitstream_size;
s->bitstream_size = buf_size;
/* do not decode until buffer has at least max_framesize bytes or
the end of the file has been reached */
* the end of the file has been reached */
if (buf_size < s->max_framesize && avpkt->data) {
*got_frame_ptr = 0;
return input_buf_size;
}
}
/* init and position bitstream reader */
init_get_bits(&s->gb, buf, buf_size*8);
init_get_bits(&s->gb, buf, buf_size * 8);
skip_bits(&s->gb, s->bitindex);
/* process header or next subblock */
@@ -471,7 +483,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
unsigned int cmd;
int len;
if (get_bits_left(&s->gb) < 3+FNSIZE) {
if (get_bits_left(&s->gb) < 3 + FNSIZE) {
*got_frame_ptr = 0;
break;
}
@@ -487,32 +499,32 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
if (!is_audio_command[cmd]) {
/* process non-audio command */
switch (cmd) {
case FN_VERBATIM:
len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
while (len--) {
get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
}
break;
case FN_BITSHIFT:
s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
break;
case FN_BLOCKSIZE: {
int blocksize = get_uint(s, av_log2(s->blocksize));
if (blocksize > s->blocksize) {
av_log(avctx, AV_LOG_ERROR, "Increasing block size is not supported\n");
return AVERROR_PATCHWELCOME;
}
if (!blocksize || blocksize > MAX_BLOCKSIZE) {
av_log(avctx, AV_LOG_ERROR, "invalid or unsupported "
"block size: %d\n", blocksize);
return AVERROR(EINVAL);
}
s->blocksize = blocksize;
break;
case FN_VERBATIM:
len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
while (len--)
get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
break;
case FN_BITSHIFT:
s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
break;
case FN_BLOCKSIZE: {
unsigned blocksize = get_uint(s, av_log2(s->blocksize));
if (blocksize > s->blocksize) {
av_log(avctx, AV_LOG_ERROR,
"Increasing block size is not supported\n");
return AVERROR_PATCHWELCOME;
}
case FN_QUIT:
s->got_quit_command = 1;
break;
if (!blocksize || blocksize > MAX_BLOCKSIZE) {
av_log(avctx, AV_LOG_ERROR, "invalid or unsupported "
"block size: %d\n", blocksize);
return AVERROR(EINVAL);
}
s->blocksize = blocksize;
break;
}
case FN_QUIT:
s->got_quit_command = 1;
break;
}
if (cmd == FN_BLOCKSIZE || cmd == FN_QUIT) {
*got_frame_ptr = 0;
@@ -538,7 +550,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
coffset = s->offset[channel][0];
else {
int32_t sum = (s->version < 2) ? 0 : s->nmean / 2;
for (i=0; i<s->nmean; i++)
for (i = 0; i < s->nmean; i++)
sum += s->offset[channel][i];
coffset = sum / s->nmean;
if (s->version >= 2)
@@ -547,21 +559,22 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
/* decode samples for this channel */
if (cmd == FN_ZERO) {
for (i=0; i<s->blocksize; i++)
for (i = 0; i < s->blocksize; i++)
s->decoded[channel][i] = 0;
} else {
if ((ret = decode_subframe_lpc(s, cmd, channel, residual_size, coffset)) < 0)
if ((ret = decode_subframe_lpc(s, cmd, channel,
residual_size, coffset)) < 0)
return ret;
}
/* update means with info from the current block */
if (s->nmean > 0) {
int32_t sum = (s->version < 2) ? 0 : s->blocksize / 2;
for (i=0; i<s->blocksize; i++)
for (i = 0; i < s->blocksize; i++)
sum += s->decoded[channel][i];
for (i=1; i<s->nmean; i++)
s->offset[channel][i-1] = s->offset[channel][i];
for (i = 1; i < s->nmean; i++)
s->offset[channel][i - 1] = s->offset[channel][i];
if (s->version < 2)
s->offset[channel][s->nmean - 1] = sum / s->blocksize;
@@ -570,11 +583,11 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
}
/* copy wrap samples for use with next block */
for (i=-s->nwrap; i<0; i++)
for (i = -s->nwrap; i < 0; i++)
s->decoded[channel][i] = s->decoded[channel][i + s->blocksize];
/* shift samples to add in unused zero bits which were removed
during encoding */
* during encoding */
fix_bitshift(s, s->decoded[channel]);
/* if this is the last channel in the block, output the samples */
@@ -617,12 +630,12 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
*got_frame_ptr = 0;
finish_frame:
s->bitindex = get_bits_count(&s->gb) - 8*((get_bits_count(&s->gb))/8);
i= (get_bits_count(&s->gb))/8;
s->bitindex = get_bits_count(&s->gb) - 8 * (get_bits_count(&s->gb) / 8);
i = get_bits_count(&s->gb) / 8;
if (i > buf_size) {
av_log(s->avctx, AV_LOG_ERROR, "overread: %d\n", i - buf_size);
s->bitstream_size=0;
s->bitstream_index=0;
s->bitstream_size = 0;
s->bitstream_index = 0;
return AVERROR_INVALIDDATA;
}
if (s->bitstream_size) {

View File

@@ -327,12 +327,13 @@ static int filter_frame(AVFilterLink *inlink, AVFilterBufferRef *in)
AVFilterLink *outlink = inlink->dst->outputs[0];
AVFilterBufferRef *out;
int direct = 0, c;
int direct, c;
if (in->perms & AV_PERM_WRITE) {
direct = 1;
out = in;
} else {
direct = 0;
out = ff_get_video_buffer(outlink, AV_PERM_WRITE, outlink->w, outlink->h);
if (!out) {
avfilter_unref_bufferp(&in);

View File

@@ -277,11 +277,9 @@ fail:
static int flac_probe(AVProbeData *p)
{
const uint8_t *bufptr = p->buf;
const uint8_t *end = p->buf + p->buf_size;
if(bufptr > end-4 || memcmp(bufptr, "fLaC", 4)) return 0;
else return AVPROBE_SCORE_MAX/2;
if (p->buf_size < 4 || memcmp(p->buf, "fLaC", 4))
return 0;
return AVPROBE_SCORE_MAX/2;
}
AVInputFormat ff_flac_demuxer = {

View File

@@ -110,9 +110,8 @@ static inline void copy(LZOContext *c, int cnt)
*/
static inline void copy_backptr(LZOContext *c, int back, int cnt)
{
register const uint8_t *src = &c->out[-back];
register uint8_t *dst = c->out;
if (src < c->out_start || src > dst) {
if (dst - c->out_start < back) {
c->error |= AV_LZO_INVALID_BACKPTR;
return;
}