From 0884dd5a1b87aff6c8a06e6492dece5cef8f3978 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 3 Oct 2011 07:37:24 -0700 Subject: [PATCH 01/15] mpegvideo: fix position of bottom edge. It was wrong in colorspaces where horizontal and vertical chroma subsampling are not the same, e.g. 422. --- libavcodec/mpegvideo.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index e418e95a33..baab3c85e6 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -2313,12 +2313,15 @@ void ff_draw_horiz_band(MpegEncContext *s, int y, int h){ edge_h= FFMIN(h, s->v_edge_pos - y); - s->dsp.draw_edges(s->current_picture_ptr->f.data[0] + y *s->linesize , s->linesize, - s->h_edge_pos , edge_h , EDGE_WIDTH , EDGE_WIDTH , sides); - s->dsp.draw_edges(s->current_picture_ptr->f.data[1] + (y>>vshift)*s->uvlinesize, s->uvlinesize, - s->h_edge_pos>>hshift, edge_h>>hshift, EDGE_WIDTH>>hshift, EDGE_WIDTH>>vshift, sides); - s->dsp.draw_edges(s->current_picture_ptr->f.data[2] + (y>>vshift)*s->uvlinesize, s->uvlinesize, - s->h_edge_pos>>hshift, edge_h>>hshift, EDGE_WIDTH>>hshift, EDGE_WIDTH>>vshift, sides); + s->dsp.draw_edges(s->current_picture_ptr->f.data[0] + y *s->linesize, + s->linesize, s->h_edge_pos, edge_h, + EDGE_WIDTH, EDGE_WIDTH, sides); + s->dsp.draw_edges(s->current_picture_ptr->f.data[1] + (y>>vshift)*s->uvlinesize, + s->uvlinesize, s->h_edge_pos>>hshift, edge_h>>vshift, + EDGE_WIDTH>>hshift, EDGE_WIDTH>>vshift, sides); + s->dsp.draw_edges(s->current_picture_ptr->f.data[2] + (y>>vshift)*s->uvlinesize, + s->uvlinesize, s->h_edge_pos>>hshift, edge_h>>vshift, + EDGE_WIDTH>>hshift, EDGE_WIDTH>>vshift, sides); } h= FFMIN(h, s->avctx->height - y); From 330deb75923675224fb9aed311d3d6ce3ec52420 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 3 Oct 2011 08:38:03 -0700 Subject: [PATCH 02/15] mpegvideo: set correct offset for edge emulation buffer. Using the old code, half of it was unused and the other half was too small for e.g. >8bpp interlaced data, causing random buffer overruns. --- libavcodec/mpegvideo.c | 6 ++---- libavcodec/mpegvideo.h | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index baab3c85e6..b8ac2ceb3d 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -377,8 +377,7 @@ static int init_duplicate_context(MpegEncContext *s, MpegEncContext *base){ int i; // edge emu needs blocksize + filter length - 1 (=17x17 for halfpel / 21x21 for h264) - FF_ALLOCZ_OR_GOTO(s->avctx, s->allocated_edge_emu_buffer, (s->width+64)*2*21*2, fail); //(width + edge + align)*interlaced*MBsize*tolerance - s->edge_emu_buffer= s->allocated_edge_emu_buffer + (s->width+64)*2*21; + FF_ALLOCZ_OR_GOTO(s->avctx, s->edge_emu_buffer, (s->width+64)*2*21*2, fail); //(width + edge + align)*interlaced*MBsize*tolerance //FIXME should be linesize instead of s->width*2 but that is not known before get_buffer() FF_ALLOCZ_OR_GOTO(s->avctx, s->me.scratchpad, (s->width+64)*4*16*2*sizeof(uint8_t), fail) @@ -416,7 +415,7 @@ fail: static void free_duplicate_context(MpegEncContext *s){ if(s==NULL) return; - av_freep(&s->allocated_edge_emu_buffer); s->edge_emu_buffer= NULL; + av_freep(&s->edge_emu_buffer); av_freep(&s->me.scratchpad); s->me.temp= s->rd_scratchpad= @@ -433,7 +432,6 @@ static void free_duplicate_context(MpegEncContext *s){ static void backup_duplicate_context(MpegEncContext *bak, MpegEncContext *src){ #define COPY(a) bak->a= src->a - COPY(allocated_edge_emu_buffer); COPY(edge_emu_buffer); COPY(me.scratchpad); COPY(me.temp); diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h index 923d46c884..d0f4bfdc6c 100644 --- a/libavcodec/mpegvideo.h +++ b/libavcodec/mpegvideo.h @@ -319,8 +319,7 @@ typedef struct MpegEncContext { uint8_t *mbintra_table; ///< used to avoid setting {ac, dc, cbp}-pred stuff to zero on inter MB decoding uint8_t *cbp_table; ///< used to store cbp, ac_pred for partitioned decoding uint8_t *pred_dir_table; ///< used to store pred_dir for partitioned decoding - uint8_t *allocated_edge_emu_buffer; - uint8_t *edge_emu_buffer; ///< points into the middle of allocated_edge_emu_buffer + uint8_t *edge_emu_buffer; ///< temporary buffer for if MVs point to out-of-frame data uint8_t *rd_scratchpad; ///< scratchpad for rate distortion mb decision uint8_t *obmc_scratchpad; uint8_t *b_scratchpad; ///< scratchpad used for writing into write only buffers From 4418aa9cb3b2f0b83748e37d2952560cf84b3611 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 3 Oct 2011 08:41:51 -0700 Subject: [PATCH 03/15] h264: correct implicit_weight for field-interlaced pictures. --- libavcodec/h264.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index e5d0ed8660..3b550cf380 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -2158,7 +2158,11 @@ static void implicit_weight_table(H264Context *h, int field){ } if(field < 0){ - cur_poc = s->current_picture_ptr->poc; + if (s->picture_structure == PICT_FRAME) { + cur_poc = s->current_picture_ptr->poc; + } else { + cur_poc = s->current_picture_ptr->field_poc[s->picture_structure - 1]; + } if( h->ref_count[0] == 1 && h->ref_count[1] == 1 && !FRAME_MBAFF && h->ref_list[0][0].poc + h->ref_list[1][0].poc == 2*cur_poc){ h->use_weight= 0; From d97efd7f879779034cca0bd16acceed47718e04e Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Wed, 5 Oct 2011 21:09:51 +0200 Subject: [PATCH 04/15] libx264: support 9- and 10-bit output. --- libavcodec/avcodec.h | 5 +++++ libavcodec/libx264.c | 29 ++++++++++++++++++++++++++++- libavcodec/utils.c | 3 +++ libavcodec/version.h | 2 +- 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index 2e37f5a1b1..5a69c24ea0 100644 --- a/libavcodec/avcodec.h +++ b/libavcodec/avcodec.h @@ -2984,6 +2984,11 @@ typedef struct AVCodec { * Private codec-specific defaults. */ const AVCodecDefault *defaults; + + /** + * Initialize codec static data, called from avcodec_register(). + */ + void (*init_static_data)(struct AVCodec *codec); } AVCodec; /** diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c index 5331bc96b6..f2c836eaef 100644 --- a/libavcodec/libx264.c +++ b/libavcodec/libx264.c @@ -20,6 +20,7 @@ */ #include "libavutil/opt.h" +#include "libavutil/pixdesc.h" #include "avcodec.h" #include "internal.h" #include @@ -123,6 +124,8 @@ static int X264_frame(AVCodecContext *ctx, uint8_t *buf, x264_picture_init( &x4->pic ); x4->pic.img.i_csp = X264_CSP_I420; + if (x264_bit_depth > 8) + x4->pic.img.i_csp |= X264_CSP_HIGH_DEPTH; x4->pic.img.i_plane = 3; if (frame) { @@ -456,6 +459,30 @@ static av_cold int X264_init(AVCodecContext *avctx) return 0; } +static const enum PixelFormat pix_fmts_8bit[] = { + PIX_FMT_YUV420P, + PIX_FMT_YUVJ420P, + PIX_FMT_NONE +}; +static const enum PixelFormat pix_fmts_9bit[] = { + PIX_FMT_YUV420P9, + PIX_FMT_NONE +}; +static const enum PixelFormat pix_fmts_10bit[] = { + PIX_FMT_YUV420P10, + PIX_FMT_NONE +}; + +static av_cold void X264_init_static(AVCodec *codec) +{ + if (x264_bit_depth == 8) + codec->pix_fmts = pix_fmts_8bit; + else if (x264_bit_depth == 9) + codec->pix_fmts = pix_fmts_9bit; + else if (x264_bit_depth == 10) + codec->pix_fmts = pix_fmts_10bit; +} + #define OFFSET(x) offsetof(X264Context, x) #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM static const AVOption options[] = { @@ -544,8 +571,8 @@ AVCodec ff_libx264_encoder = { .encode = X264_frame, .close = X264_close, .capabilities = CODEC_CAP_DELAY, - .pix_fmts = (const enum PixelFormat[]) { PIX_FMT_YUV420P, PIX_FMT_YUVJ420P, PIX_FMT_NONE }, .long_name = NULL_IF_CONFIG_SMALL("libx264 H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10"), .priv_class = &class, .defaults = x264_defaults, + .init_static_data = X264_init_static, }; diff --git a/libavcodec/utils.c b/libavcodec/utils.c index b598942354..1c95fa1236 100644 --- a/libavcodec/utils.c +++ b/libavcodec/utils.c @@ -107,6 +107,9 @@ void avcodec_register(AVCodec *codec) while (*p != NULL) p = &(*p)->next; *p = codec; codec->next = NULL; + + if (codec->init_static_data) + codec->init_static_data(codec); } unsigned avcodec_get_edge_width(void) diff --git a/libavcodec/version.h b/libavcodec/version.h index d745fae543..d9e60cca01 100644 --- a/libavcodec/version.h +++ b/libavcodec/version.h @@ -22,7 +22,7 @@ #define LIBAVCODEC_VERSION_MAJOR 53 #define LIBAVCODEC_VERSION_MINOR 12 -#define LIBAVCODEC_VERSION_MICRO 0 +#define LIBAVCODEC_VERSION_MICRO 1 #define LIBAVCODEC_VERSION_INT AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \ LIBAVCODEC_VERSION_MINOR, \ From 69a0bce753a5d5556d5bc0888afe390e22611dd8 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sat, 10 Sep 2011 13:28:13 +0200 Subject: [PATCH 05/15] Fixed deference of NULL pointer in motionpixels decoder. Some of the arguments given to init_vlc() come from the stream and can be corrupted. Signed-off-by: Janne Grunau --- libavcodec/motionpixels.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/motionpixels.c b/libavcodec/motionpixels.c index 21e1bda99c..9589b4c386 100644 --- a/libavcodec/motionpixels.c +++ b/libavcodec/motionpixels.c @@ -278,7 +278,8 @@ static int mp_decode_frame(AVCodecContext *avctx, if (sz == 0) goto end; - init_vlc(&mp->vlc, mp->max_codes_bits, mp->codes_count, &mp->codes[0].size, sizeof(HuffCode), 1, &mp->codes[0].code, sizeof(HuffCode), 4, 0); + if (init_vlc(&mp->vlc, mp->max_codes_bits, mp->codes_count, &mp->codes[0].size, sizeof(HuffCode), 1, &mp->codes[0].code, sizeof(HuffCode), 4, 0)) + goto end; mp_decode_frame_helper(mp, &gb); free_vlc(&mp->vlc); From 790f4dd5c96f948a637a90899b5f6af34ccb3942 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 11 Sep 2011 19:17:41 +0200 Subject: [PATCH 06/15] Fixed segfault on corrupted sega streams in the demuxer. Signed-off-by: Janne Grunau --- libavformat/segafilm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libavformat/segafilm.c b/libavformat/segafilm.c index 81f04c6c01..32a979f1a3 100644 --- a/libavformat/segafilm.c +++ b/libavformat/segafilm.c @@ -176,6 +176,8 @@ static int film_read_header(AVFormatContext *s, if(film->sample_count >= UINT_MAX / sizeof(film_sample)) return -1; film->sample_table = av_malloc(film->sample_count * sizeof(film_sample)); + if (!film->sample_table) + return AVERROR(ENOMEM); for(i=0; inb_streams; i++) av_set_pts_info(s->streams[i], 33, 1, film->base_clock); @@ -199,7 +201,7 @@ static int film_read_header(AVFormatContext *s, if (film->audio_type == CODEC_ID_ADPCM_ADX) audio_frame_counter += (film->sample_table[i].sample_size * 32 / (18 * film->audio_channels)); - else + else if (film->audio_type != CODEC_ID_NONE) audio_frame_counter += (film->sample_table[i].sample_size / (film->audio_channels * film->audio_bits / 8)); } else { From 762ffa6861a8afefe11e1fe18e7b47c18d2d62f6 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Mon, 12 Sep 2011 21:09:57 +0200 Subject: [PATCH 07/15] segafilm: Fix potential division by 0 on corrupted streams in the demuxer Signed-off-by: Janne Grunau --- libavformat/segafilm.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/libavformat/segafilm.c b/libavformat/segafilm.c index 32a979f1a3..ea464f29fd 100644 --- a/libavformat/segafilm.c +++ b/libavformat/segafilm.c @@ -113,11 +113,14 @@ static int film_read_header(AVFormatContext *s, film->audio_bits = scratch[22]; if (scratch[23] == 2) film->audio_type = CODEC_ID_ADPCM_ADX; - else if (film->audio_bits == 8) - film->audio_type = CODEC_ID_PCM_S8; - else if (film->audio_bits == 16) - film->audio_type = CODEC_ID_PCM_S16BE; - else + else if (film->audio_channels > 0) { + if (film->audio_bits == 8) + film->audio_type = CODEC_ID_PCM_S8; + else if (film->audio_bits == 16) + film->audio_type = CODEC_ID_PCM_S16BE; + else + film->audio_type = CODEC_ID_NONE; + } else film->audio_type = CODEC_ID_NONE; } From 1775b92fee43f0527e2f5892a5a30450fa929722 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Thu, 6 Oct 2011 22:53:41 +0200 Subject: [PATCH 08/15] segafilm: Check for memory allocation failures in segafilm demuxer. Signed-off-by: Janne Grunau --- libavformat/segafilm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavformat/segafilm.c b/libavformat/segafilm.c index ea464f29fd..4bbacc5fa6 100644 --- a/libavformat/segafilm.c +++ b/libavformat/segafilm.c @@ -257,6 +257,10 @@ static int film_read_packet(AVFormatContext *s, av_free(film->stereo_buffer); film->stereo_buffer_size = sample->sample_size; film->stereo_buffer = av_malloc(film->stereo_buffer_size); + if (!film->stereo_buffer) { + film->stereo_buffer_size = 0; + return AVERROR(ENOMEM); + } } pkt->pos= avio_tell(pb); From 3a742470a845c24e7c3a40c0a228705ca951e673 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sat, 17 Sep 2011 16:56:35 +0200 Subject: [PATCH 09/15] cook: Fix js_vlc_bits value validation for joint stereo Signed-off-by: Janne Grunau --- libavcodec/cook.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libavcodec/cook.c b/libavcodec/cook.c index b5492db27b..13f2dd66a6 100644 --- a/libavcodec/cook.c +++ b/libavcodec/cook.c @@ -1174,8 +1174,9 @@ static av_cold int cook_decode_init(AVCodecContext *avctx) return -1; } - if ((q->subpacket[s].js_vlc_bits > 6) || (q->subpacket[s].js_vlc_bits < 0)) { - av_log(avctx,AV_LOG_ERROR,"js_vlc_bits = %d, only >= 0 and <= 6 allowed!\n",q->subpacket[s].js_vlc_bits); + if ((q->subpacket[s].js_vlc_bits > 6) || (q->subpacket[s].js_vlc_bits < 2*q->subpacket[s].joint_stereo)) { + av_log(avctx,AV_LOG_ERROR,"js_vlc_bits = %d, only >= %d and <= 6 allowed!\n", + q->subpacket[s].js_vlc_bits, 2*q->subpacket[s].joint_stereo); return -1; } From c0cbe36b18ab3eb13a53fe684ec1f63a00df2c86 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 11 Sep 2011 19:17:45 +0200 Subject: [PATCH 10/15] vmd: fix segfaults on corruped streams Signed-off-by: Janne Grunau --- libavcodec/vmdav.c | 104 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 24 deletions(-) diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c index 45289966b7..772f98c70f 100644 --- a/libavcodec/vmdav.c +++ b/libavcodec/vmdav.c @@ -72,9 +72,11 @@ typedef struct VmdVideoContext { #define QUEUE_SIZE 0x1000 #define QUEUE_MASK 0x0FFF -static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_len) +static void lz_unpack(const unsigned char *src, int src_len, + unsigned char *dest, int dest_len) { const unsigned char *s; + unsigned int s_len; unsigned char *d; unsigned char *d_end; unsigned char queue[QUEUE_SIZE]; @@ -87,13 +89,16 @@ static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_le unsigned int i, j; s = src; + s_len = src_len; d = dest; d_end = d + dest_len; dataleft = AV_RL32(s); - s += 4; + s += 4; s_len -= 4; memset(queue, 0x20, QUEUE_SIZE); + if (s_len < 4) + return; if (AV_RL32(s) == 0x56781234) { - s += 4; + s += 4; s_len -= 4; qpos = 0x111; speclen = 0xF + 3; } else { @@ -101,32 +106,41 @@ static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_le speclen = 100; /* no speclen */ } - while (dataleft > 0) { - tag = *s++; + while (dataleft > 0 && s_len > 0) { + tag = *s++; s_len--; if ((tag == 0xFF) && (dataleft > 8)) { - if (d + 8 > d_end) + if (d + 8 > d_end || s_len < 8) return; for (i = 0; i < 8; i++) { queue[qpos++] = *d++ = *s++; qpos &= QUEUE_MASK; } + s_len -= 8; dataleft -= 8; } else { for (i = 0; i < 8; i++) { if (dataleft == 0) break; if (tag & 0x01) { - if (d + 1 > d_end) + if (d + 1 > d_end || s_len < 1) return; queue[qpos++] = *d++ = *s++; qpos &= QUEUE_MASK; dataleft--; + s_len--; } else { + if (s_len < 2) + return; chainofs = *s++; chainofs |= ((*s & 0xF0) << 4); chainlen = (*s++ & 0x0F) + 3; - if (chainlen == speclen) + s_len -= 2; + if (chainlen == speclen) { + if (s_len < 1) + return; chainlen = *s++ + 0xF + 3; + s_len--; + } if (d + chainlen > d_end) return; for (j = 0; j < chainlen; j++) { @@ -143,7 +157,7 @@ static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_le } static int rle_unpack(const unsigned char *src, unsigned char *dest, - int src_len, int dest_len) + int src_count, int src_size, int dest_len) { const unsigned char *ps; unsigned char *pd; @@ -152,31 +166,40 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest, ps = src; pd = dest; - if (src_len & 1) + if (src_count & 1) { + if (src_size < 1) + return 0; *pd++ = *ps++; + src_size--; + } - src_len >>= 1; + src_count >>= 1; i = 0; do { + if (src_size < 1) + break; l = *ps++; + src_size--; if (l & 0x80) { l = (l & 0x7F) * 2; - if (pd + l > dest_end) + if (pd + l > dest_end || src_size < l) return ps - src; memcpy(pd, ps, l); ps += l; + src_size -= l; pd += l; } else { - if (pd + i > dest_end) + if (pd + i > dest_end || src_size < 2) return ps - src; for (i = 0; i < l; i++) { *pd++ = ps[0]; *pd++ = ps[1]; } ps += 2; + src_size -= 2; } i += l; - } while (i < src_len); + } while (i < src_count); return ps - src; } @@ -191,6 +214,7 @@ static void vmd_decode(VmdVideoContext *s) const unsigned char *p = s->buf + 16; const unsigned char *pb; + unsigned int pb_size; unsigned char meth; unsigned char *dp; /* pointer to current frame */ unsigned char *pp; /* pointer to previous frame */ @@ -204,6 +228,16 @@ static void vmd_decode(VmdVideoContext *s) frame_y = AV_RL16(&s->buf[8]); frame_width = AV_RL16(&s->buf[10]) - frame_x + 1; frame_height = AV_RL16(&s->buf[12]) - frame_y + 1; + if (frame_x < 0 || frame_width < 0 || + frame_x >= s->avctx->width || + frame_width > s->avctx->width || + frame_x + frame_width > s->avctx->width) + return; + if (frame_y < 0 || frame_height < 0 || + frame_y >= s->avctx->height || + frame_height > s->avctx->height || + frame_y + frame_height > s->avctx->height) + return; if ((frame_width == s->avctx->width && frame_height == s->avctx->height) && (frame_x || frame_y)) { @@ -216,8 +250,9 @@ static void vmd_decode(VmdVideoContext *s) /* if only a certain region will be updated, copy the entire previous * frame before the decode */ - if (frame_x || frame_y || (frame_width != s->avctx->width) || - (frame_height != s->avctx->height)) { + if (s->prev_frame.data[0] && + (frame_x || frame_y || (frame_width != s->avctx->width) || + (frame_height != s->avctx->height))) { memcpy(s->frame.data[0], s->prev_frame.data[0], s->avctx->height * s->frame.linesize[0]); @@ -235,14 +270,19 @@ static void vmd_decode(VmdVideoContext *s) } s->size -= (256 * 3 + 2); } - if (s->size >= 0) { + if (s->size > 0) { /* originally UnpackFrame in VAG's code */ pb = p; - meth = *pb++; + pb_size = s->buf + s->size - pb; + if (pb_size < 1) + return; + meth = *pb++; pb_size--; if (meth & 0x80) { - lz_unpack(pb, s->unpack_buffer, s->unpack_buffer_size); + lz_unpack(pb, pb_size, + s->unpack_buffer, s->unpack_buffer_size); meth &= 0x7F; pb = s->unpack_buffer; + pb_size = s->unpack_buffer_size; } dp = &s->frame.data[0][frame_y * s->frame.linesize[0] + frame_x]; @@ -252,17 +292,21 @@ static void vmd_decode(VmdVideoContext *s) for (i = 0; i < frame_height; i++) { ofs = 0; do { + if (pb_size < 1) + return; len = *pb++; + pb_size--; if (len & 0x80) { len = (len & 0x7F) + 1; - if (ofs + len > frame_width) + if (ofs + len > frame_width || pb_size < len) return; memcpy(&dp[ofs], pb, len); pb += len; + pb_size -= len; ofs += len; } else { /* interframe pixel copy */ - if (ofs + len + 1 > frame_width) + if (ofs + len + 1 > frame_width || !s->prev_frame.data[0]) return; memcpy(&dp[ofs], &pp[ofs], len + 1); ofs += len + 1; @@ -280,8 +324,11 @@ static void vmd_decode(VmdVideoContext *s) case 2: for (i = 0; i < frame_height; i++) { + if (pb_size < frame_width) + return; memcpy(dp, pb, frame_width); pb += frame_width; + pb_size -= frame_width; dp += s->frame.linesize[0]; pp += s->prev_frame.linesize[0]; } @@ -291,18 +338,27 @@ static void vmd_decode(VmdVideoContext *s) for (i = 0; i < frame_height; i++) { ofs = 0; do { + if (pb_size < 1) + return; len = *pb++; + pb_size--; if (len & 0x80) { len = (len & 0x7F) + 1; + if (pb_size < 1) + return; if (*pb++ == 0xFF) - len = rle_unpack(pb, &dp[ofs], len, frame_width - ofs); - else + len = rle_unpack(pb, &dp[ofs], len, pb_size, frame_width - ofs); + else { + if (pb_size < len) + return; memcpy(&dp[ofs], pb, len); + } pb += len; + pb_size -= 1 + len; ofs += len; } else { /* interframe pixel copy */ - if (ofs + len + 1 > frame_width) + if (ofs + len + 1 > frame_width || !s->prev_frame.data[0]) return; memcpy(&dp[ofs], &pp[ofs], len + 1); ofs += len + 1; From d239d4b447885cb7c5eee9ce359f34ad6b64f373 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Sun, 11 Sep 2011 19:17:43 +0200 Subject: [PATCH 11/15] cinepak: Fix invalid read access on extra data Signed-off-by: Janne Grunau --- libavcodec/cinepak.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/cinepak.c b/libavcodec/cinepak.c index 1d41ba2287..c5d47be2ac 100644 --- a/libavcodec/cinepak.c +++ b/libavcodec/cinepak.c @@ -336,7 +336,8 @@ static int cinepak_decode (CinepakContext *s) * If the frame header is followed by the bytes FE 00 00 06 00 00 then * this is probably one of the two known files that have 6 extra bytes * after the frame header. Else, assume 2 extra bytes. */ - if ((s->data[10] == 0xFE) && + if (s->size >= 16 && + (s->data[10] == 0xFE) && (s->data[11] == 0x00) && (s->data[12] == 0x00) && (s->data[13] == 0x06) && From 0ec6d6e9b682318b5b5b5457e09fbf3c4ca41335 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Wed, 21 Sep 2011 20:46:30 +0200 Subject: [PATCH 12/15] vp56: Check for missing reference frame data Signed-off-by: Janne Grunau --- libavcodec/vp56.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c index 5c7e93e5e6..530b890de1 100644 --- a/libavcodec/vp56.c +++ b/libavcodec/vp56.c @@ -401,6 +401,8 @@ static void vp56_decode_mb(VP56Context *s, int row, int col, int is_alpha) frame_current = s->framep[VP56_FRAME_CURRENT]; frame_ref = s->framep[ref_frame]; + if (mb_type != VP56_MB_INTRA && !frame_ref->data[0]) + return; ab = 6*is_alpha; b_max = 6 - 2*is_alpha; From 066fff755a5d8edc660c010ddb08474d208eeade Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Wed, 21 Sep 2011 20:46:32 +0200 Subject: [PATCH 13/15] vp6: Check for huffman tree build errors Signed-off-by: Janne Grunau --- libavcodec/vp5.c | 3 ++- libavcodec/vp56.c | 4 +++- libavcodec/vp56.h | 2 +- libavcodec/vp6.c | 20 ++++++++++++-------- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/libavcodec/vp5.c b/libavcodec/vp5.c index 7743a48508..a1a38b0a03 100644 --- a/libavcodec/vp5.c +++ b/libavcodec/vp5.c @@ -118,7 +118,7 @@ static void vp5_parse_vector_models(VP56Context *s) model->vector_pdv[comp][node] = vp56_rac_gets_nn(c, 7); } -static void vp5_parse_coeff_models(VP56Context *s) +static int vp5_parse_coeff_models(VP56Context *s) { VP56RangeCoder *c = &s->c; VP56Model *model = s->modelp; @@ -162,6 +162,7 @@ static void vp5_parse_coeff_models(VP56Context *s) for (ctx=0; ctx<6; ctx++) for (node=0; node<5; node++) model->coeff_acct[pt][ct][cg][ctx][node] = av_clip(((model->coeff_ract[pt][ct][cg][node] * vp5_ract_lc[ct][cg][node][ctx][0] + 128) >> 8) + vp5_ract_lc[ct][cg][node][ctx][1], 1, 254); + return 0; } static void vp5_parse_coeff(VP56Context *s) diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c index 530b890de1..e18c59ef66 100644 --- a/libavcodec/vp56.c +++ b/libavcodec/vp56.c @@ -541,7 +541,8 @@ int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size, s->mb_type = VP56_MB_INTER_NOVEC_PF; } - s->parse_coeff_models(s); + if (s->parse_coeff_models(s)) + goto next; memset(s->prev_dc, 0, sizeof(s->prev_dc)); s->prev_dc[1][VP56_FRAME_CURRENT] = 128; @@ -605,6 +606,7 @@ int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size, } } + next: if (p->key_frame || golden_frame) { if (s->framep[VP56_FRAME_GOLDEN]->data[0] && s->framep[VP56_FRAME_GOLDEN] != s->framep[VP56_FRAME_GOLDEN2]) diff --git a/libavcodec/vp56.h b/libavcodec/vp56.h index 8bb7251fc6..ceb516d88d 100644 --- a/libavcodec/vp56.h +++ b/libavcodec/vp56.h @@ -48,7 +48,7 @@ typedef void (*VP56Filter)(VP56Context *s, uint8_t *dst, uint8_t *src, typedef void (*VP56ParseCoeff)(VP56Context *s); typedef void (*VP56DefaultModelsInit)(VP56Context *s); typedef void (*VP56ParseVectorModels)(VP56Context *s); -typedef void (*VP56ParseCoeffModels)(VP56Context *s); +typedef int (*VP56ParseCoeffModels)(VP56Context *s); typedef int (*VP56ParseHeader)(VP56Context *s, const uint8_t *buf, int buf_size, int *golden_frame); diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c index 657a5da7aa..6928a91a05 100644 --- a/libavcodec/vp6.c +++ b/libavcodec/vp6.c @@ -236,7 +236,7 @@ static int vp6_build_huff_tree(VP56Context *s, uint8_t coeff_model[], FF_HUFFMAN_FLAG_HNODE_FIRST); } -static void vp6_parse_coeff_models(VP56Context *s) +static int vp6_parse_coeff_models(VP56Context *s) { VP56RangeCoder *c = &s->c; VP56Model *model = s->modelp; @@ -281,15 +281,18 @@ static void vp6_parse_coeff_models(VP56Context *s) if (s->use_huffman) { for (pt=0; pt<2; pt++) { - vp6_build_huff_tree(s, model->coeff_dccv[pt], - vp6_huff_coeff_map, 12, &s->dccv_vlc[pt]); - vp6_build_huff_tree(s, model->coeff_runv[pt], - vp6_huff_run_map, 9, &s->runv_vlc[pt]); + if (vp6_build_huff_tree(s, model->coeff_dccv[pt], + vp6_huff_coeff_map, 12, &s->dccv_vlc[pt])) + return -1; + if (vp6_build_huff_tree(s, model->coeff_runv[pt], + vp6_huff_run_map, 9, &s->runv_vlc[pt])) + return -1; for (ct=0; ct<3; ct++) for (cg = 0; cg < 6; cg++) - vp6_build_huff_tree(s, model->coeff_ract[pt][ct][cg], - vp6_huff_coeff_map, 12, - &s->ract_vlc[pt][ct][cg]); + if (vp6_build_huff_tree(s, model->coeff_ract[pt][ct][cg], + vp6_huff_coeff_map, 12, + &s->ract_vlc[pt][ct][cg])) + return -1; } memset(s->nb_null, 0, sizeof(s->nb_null)); } else { @@ -299,6 +302,7 @@ static void vp6_parse_coeff_models(VP56Context *s) for (node=0; node<5; node++) model->coeff_dcct[pt][ctx][node] = av_clip(((model->coeff_dccv[pt][node] * vp6_dccv_lc[ctx][node][0] + 128) >> 8) + vp6_dccv_lc[ctx][node][1], 1, 255); } + return 0; } static void vp6_parse_vector_adjustment(VP56Context *s, VP56mv *vect) From 3d09d0017d10a0d738141a955c75c555133e41b2 Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Wed, 21 Sep 2011 20:46:33 +0200 Subject: [PATCH 14/15] vp56: Release old pictures after a resolution changes Signed-off-by: Janne Grunau --- libavcodec/vp56.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c index e18c59ef66..3b2ac95837 100644 --- a/libavcodec/vp56.c +++ b/libavcodec/vp56.c @@ -467,6 +467,7 @@ static int vp56_size_changed(AVCodecContext *avctx) s->mb_height = (avctx->coded_height+15) / 16; if (s->mb_width > 1000 || s->mb_height > 1000) { + avcodec_set_dimensions(avctx, 0, 0); av_log(avctx, AV_LOG_ERROR, "picture too big\n"); return -1; } @@ -515,6 +516,18 @@ int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size, if (!res) return -1; + if (res == 2) { + int i; + for (i = 0; i < 4; i++) { + if (s->frames[i].data[0]) + avctx->release_buffer(avctx, &s->frames[i]); + } + if (is_alpha) { + avcodec_set_dimensions(avctx, 0, 0); + return -1; + } + } + if (!is_alpha) { p->reference = 1; if (avctx->get_buffer(avctx, p) < 0) { From a72cad0a6c05aa74940101e937cb3dc602d7d67b Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Fri, 23 Sep 2011 22:36:11 +0200 Subject: [PATCH 15/15] vp6: Reset the internal state when aborting key frames header parsing It prevents leaving the state only half initialized. Signed-off-by: Janne Grunau --- libavcodec/vp6.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c index 6928a91a05..e29f9014aa 100644 --- a/libavcodec/vp6.c +++ b/libavcodec/vp6.c @@ -139,8 +139,11 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size, if (coeff_offset) { buf += coeff_offset; buf_size -= coeff_offset; - if (buf_size < 0) + if (buf_size < 0) { + if (s->framep[VP56_FRAME_CURRENT]->key_frame) + avcodec_set_dimensions(s->avctx, 0, 0); return 0; + } if (s->use_huffman) { s->parse_coeff = vp6_parse_coeff_huffman; init_get_bits(&s->gb, buf, buf_size<<3);