From b102d5d97daedb717c023ec7bfa43047d97de284 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Wed, 30 May 2012 04:04:54 +0100 Subject: [PATCH 01/12] h264: allow cropping to AVCodecContext.width/height Override the frame size from the SPS with AVCodecContext values if the latter specify a size smaller by less than one macroblock. This is required for correct cropping of MOV files from Canon cameras. Signed-off-by: Mans Rullgard (cherry picked from commit 30f515091c323da59c0f1b533703dedca2f4b95d) Conflicts: libavcodec/h264.c (cherry picked from commit e1608014c50eeb9f4744a53de0794eb6bb1269a2) Signed-off-by: Reinhard Tartler --- libavcodec/h264.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index beb89a07e1..90293f17f4 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -2599,6 +2599,12 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ else s->height= 16*s->mb_height - (4>>CHROMA444)*FFMIN(h->sps.crop_bottom, (8<avctx->width, 16) == s->width && + FFALIGN(s->avctx->height, 16) == s->height) { + s->width = s->avctx->width; + s->height = s->avctx->height; + } + if (s->context_initialized && ( s->width != s->avctx->width || s->height != s->avctx->height || av_cmp_q(h->sps.sar, s->avctx->sample_aspect_ratio))) { From 0054d70f23edd1f61a10a1c2c687b3a04831feb9 Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Wed, 30 May 2012 04:06:00 +0100 Subject: [PATCH 02/12] mov: set AVCodecContext.width/height for h264 This is required for correct cropping of files from Canon cameras. Signed-off-by: Mans Rullgard (cherry picked from commit 8aa93e900449c88c3169ff5636fed03f41779cac) Signed-off-by: Reinhard Tartler (cherry picked from commit 2fb4be9a99a2c2a9435339830e3d940171cc0d9b) Signed-off-by: Reinhard Tartler --- libavformat/mov.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/libavformat/mov.c b/libavformat/mov.c index c720440472..b1458775aa 100644 --- a/libavformat/mov.c +++ b/libavformat/mov.c @@ -1818,9 +1818,6 @@ static int mov_read_trak(MOVContext *c, AVIOContext *pb, MOVAtom atom) #if CONFIG_H263_DECODER case CODEC_ID_H263: #endif -#if CONFIG_H264_DECODER - case CODEC_ID_H264: -#endif #if CONFIG_MPEG4_DECODER case CODEC_ID_MPEG4: #endif From a60eb6ef12df472554e93916aaeae729b0c8db03 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 6 Oct 2012 09:21:29 +0200 Subject: [PATCH 03/12] ffmpeg: fix -force_key_frames Based on commit 19ad567311b29a42e308317b5329218c590afac8 in master. --- ffmpeg.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ffmpeg.c b/ffmpeg.c index 76d1cf363c..30ff5382ba 100644 --- a/ffmpeg.c +++ b/ffmpeg.c @@ -298,6 +298,7 @@ typedef struct AVOutputStream { #endif int sws_flags; + char *forced_key_frames; } AVOutputStream; static AVOutputStream **output_streams_for_file[MAX_FILES] = { NULL }; @@ -2254,6 +2255,9 @@ static int transcode(AVFormatContext **output_files, } codec->time_base = (AVRational){ost->frame_rate.den, ost->frame_rate.num}; + if (ost->forced_key_frames) + parse_forced_key_frames(ost->forced_key_frames, ost, codec); + #if CONFIG_AVFILTER if (configure_video_filters(ist, ost)) { fprintf(stderr, "Error opening filters!\n"); @@ -2729,6 +2733,7 @@ static int transcode(AVFormatContext **output_files, av_freep(&ost->st->codec->subtitle_header); av_free(ost->pict_tmp.data[0]); av_free(ost->forced_kf_pts); + av_free(ost->forced_key_frames); if (ost->video_resample) sws_freeContext(ost->img_resample_ctx); if (ost->resample) @@ -3550,8 +3555,10 @@ static void new_video_stream(AVFormatContext *oc, int file_idx) } } - if (forced_key_frames) - parse_forced_key_frames(forced_key_frames, ost, video_enc); + if (forced_key_frames) { + ost->forced_key_frames = forced_key_frames; + forced_key_frames = NULL; + } } if (video_language) { av_dict_set(&st->metadata, "language", video_language, 0); @@ -3561,7 +3568,6 @@ static void new_video_stream(AVFormatContext *oc, int file_idx) /* reset some key parameters */ video_disable = 0; av_freep(&video_codec_name); - av_freep(&forced_key_frames); video_stream_copy = 0; frame_pix_fmt = PIX_FMT_NONE; } From 0173a7966b331105158a88f96b9afcc431d2fef8 Mon Sep 17 00:00:00 2001 From: Kostya Shishkov Date: Thu, 27 Sep 2012 19:25:06 +0200 Subject: [PATCH 04/12] vc1dec: add flush function for WMV9 and VC-1 decoders CC: libav-stable@libav.org (cherry picked from commit 4dc8c8386eef942dba35c4f2fb3210e22b511a5b) Signed-off-by: Anton Khirnov (cherry picked from commit 02b72394627933dc8ce26445231a69f00dba491b) Conflicts: libavcodec/vc1dec.c Signed-off-by: Anton Khirnov --- libavcodec/vc1dec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c index c87558bc50..1474e9fedd 100644 --- a/libavcodec/vc1dec.c +++ b/libavcodec/vc1dec.c @@ -3831,6 +3831,7 @@ AVCodec ff_vc1_decoder = { vc1_decode_frame, CODEC_CAP_DR1 | CODEC_CAP_DELAY, NULL, + .flush = ff_mpeg_flush, .long_name = NULL_IF_CONFIG_SMALL("SMPTE VC-1"), .pix_fmts = ff_hwaccel_pixfmt_list_420, .profiles = NULL_IF_CONFIG_SMALL(profiles) @@ -3848,6 +3849,7 @@ AVCodec ff_wmv3_decoder = { vc1_decode_frame, CODEC_CAP_DR1 | CODEC_CAP_DELAY, NULL, + .flush = ff_mpeg_flush, .long_name = NULL_IF_CONFIG_SMALL("Windows Media Video 9"), .pix_fmts = ff_hwaccel_pixfmt_list_420, .profiles = NULL_IF_CONFIG_SMALL(profiles) From f31170d4e7f9671e019315391160d454b18d7296 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 6 Aug 2012 13:59:04 +0200 Subject: [PATCH 05/12] nuv: check RTjpeg header for validity CC: libav-stable@libav.org (cherry picked from commit 859a579e9bbf47fae2e09494c43bcf813dcb2fad) Signed-off-by: Anton Khirnov (cherry picked from commit 6704522ca9dd32c858ee474492be568c386910f9) Signed-off-by: Anton Khirnov --- libavcodec/nuv.c | 9 +++++---- libavcodec/rtjpeg.h | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/libavcodec/nuv.c b/libavcodec/nuv.c index 0c5e42fe78..00767c5873 100644 --- a/libavcodec/nuv.c +++ b/libavcodec/nuv.c @@ -184,17 +184,18 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, } if (c->codec_frameheader) { int w, h, q; - if (buf_size < 12) { + if (buf_size < RTJPEG_HEADER_SIZE || buf[4] != RTJPEG_HEADER_SIZE || + buf[5] != RTJPEG_FILE_VERSION) { av_log(avctx, AV_LOG_ERROR, "invalid nuv video frame\n"); - return -1; + return AVERROR_INVALIDDATA; } w = AV_RL16(&buf[6]); h = AV_RL16(&buf[8]); q = buf[10]; if (!codec_reinit(avctx, w, h, q)) return -1; - buf = &buf[12]; - buf_size -= 12; + buf = &buf[RTJPEG_HEADER_SIZE]; + buf_size -= RTJPEG_HEADER_SIZE; } if (keyframe && c->pic.data[0]) diff --git a/libavcodec/rtjpeg.h b/libavcodec/rtjpeg.h index d537c93ff4..4b46689f9c 100644 --- a/libavcodec/rtjpeg.h +++ b/libavcodec/rtjpeg.h @@ -25,6 +25,9 @@ #include #include "dsputil.h" +#define RTJPEG_FILE_VERSION 0 +#define RTJPEG_HEADER_SIZE 12 + typedef struct { int w, h; DSPContext *dsp; From 8812b5f164109553f009ce385e17a1af16b6ea53 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 2 Jul 2012 10:46:39 +0200 Subject: [PATCH 06/12] imgconvert: avoid undefined left shift in avcodec_find_best_pix_fmt CC: libav-stable@libav.org (cherry picked from commit 39bb27bf79bc4c2d8beaed637a14176264cb1916) Signed-off-by: Anton Khirnov (cherry picked from commit 7a7229b52d1900279041991fadbd29b27e8dfe95) Signed-off-by: Anton Khirnov --- libavcodec/imgconvert.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c index 351ed7ada7..8da11a5c98 100644 --- a/libavcodec/imgconvert.c +++ b/libavcodec/imgconvert.c @@ -615,7 +615,8 @@ static enum PixelFormat avcodec_find_best_pix_fmt1(int64_t pix_fmt_mask, /* find exact color match with smallest size */ dst_pix_fmt = PIX_FMT_NONE; min_dist = 0x7fffffff; - for(i = 0;i < PIX_FMT_NB; i++) { + /* test only the first 64 pixel formats to avoid undefined behaviour */ + for (i = 0; i < 64; i++) { if (pix_fmt_mask & (1ULL << i)) { loss = avcodec_get_pix_fmt_loss(i, src_pix_fmt, has_alpha) & loss_mask; if (loss == 0) { From 899d95efe12f1e250b361837c1c8c06df9ac9b86 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 20 Nov 2011 17:19:25 +0100 Subject: [PATCH 07/12] mpegvideo: Don't use ff_mspel_motion() for vc1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using ff_mspel_motion assumes that s (a MpegEncContext poiinter) really is a Wmv2Context. This fixes crashes in error resilience on vc1/wmv3 videos. CC: libav-stable@libav.org Signed-off-by: Martin Storsjö (cherry picked from commit 18f2d5cb9c48d06895960f37467576725c9dc2d1) Signed-off-by: Anton Khirnov (cherry picked from commit da0c457663479bc1828918e1bb3e4a5e4de0d557) Signed-off-by: Anton Khirnov --- libavcodec/mpegvideo_common.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/mpegvideo_common.h b/libavcodec/mpegvideo_common.h index d0093ce8d2..dfedc6bade 100644 --- a/libavcodec/mpegvideo_common.h +++ b/libavcodec/mpegvideo_common.h @@ -725,7 +725,8 @@ static av_always_inline void MPV_motion_internal(MpegEncContext *s, 0, 0, 0, ref_picture, pix_op, qpix_op, s->mv[dir][0][0], s->mv[dir][0][1], 16); - }else if(!is_mpeg12 && (CONFIG_WMV2_DECODER || CONFIG_WMV2_ENCODER) && s->mspel){ + } else if (!is_mpeg12 && (CONFIG_WMV2_DECODER || CONFIG_WMV2_ENCODER) && + s->mspel && s->codec_id == CODEC_ID_WMV2) { ff_mspel_motion(s, dest_y, dest_cb, dest_cr, ref_picture, pix_op, s->mv[dir][0][0], s->mv[dir][0][1], 16); From 77d43bf42d76a1a3ccd8ea25937664c7f9d4bcac Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Fri, 15 Jun 2012 19:58:11 +0200 Subject: [PATCH 08/12] lavf: don't segfault when a NULL filename is passed to avformat_open_input() This can easily happen when the caller is using a custom AVIOContext. Behave as if the filename was an empty string in this case. CC: libav-stable@libav.org (cherry picked from commit a5db8e4a1a5449cc7a61e963c9fa698a4f22131b) Signed-off-by: Anton Khirnov (cherry picked from commit 7124fa5d3640e5b8089dd13b22a09038b2ec5216) Signed-off-by: Anton Khirnov --- libavformat/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/utils.c b/libavformat/utils.c index 57fc836c64..5f3da495fd 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -622,7 +622,7 @@ int avformat_open_input(AVFormatContext **ps, const char *filename, AVInputForma } s->duration = s->start_time = AV_NOPTS_VALUE; - av_strlcpy(s->filename, filename, sizeof(s->filename)); + av_strlcpy(s->filename, filename ? filename : "", sizeof(s->filename)); /* allocate private data */ if (s->iformat->priv_data_size > 0) { From b6ba39f931a82561389553ab2ac0ddb61b73a1e3 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 24 Mar 2012 01:39:13 +0100 Subject: [PATCH 09/12] alsdec: check opt_order. Fixes out of array write in quant_cof. Also make sure no invalid opt_order stays in the context. Fixes CVE-2012-2775 Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer Signed-off-by: Justin Ruggles (cherry picked from commit 9853e41aa0a6cfff629ff7009685eb8bf8d64e7f) Signed-off-by: Anton Khirnov (cherry picked from commit a1b127515bb79c715933d0d4201e4ef3152b3dcb) Signed-off-by: Anton Khirnov --- libavcodec/alsdec.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libavcodec/alsdec.c b/libavcodec/alsdec.c index f2944c6038..cdf2a7c0f1 100644 --- a/libavcodec/alsdec.c +++ b/libavcodec/alsdec.c @@ -662,6 +662,11 @@ static int read_var_block_data(ALSDecContext *ctx, ALSBlockData *bd) int opt_order_length = av_ceil_log2(av_clip((bd->block_length >> 3) - 1, 2, sconf->max_order + 1)); *bd->opt_order = get_bits(gb, opt_order_length); + if (*bd->opt_order > sconf->max_order) { + *bd->opt_order = sconf->max_order; + av_log(avctx, AV_LOG_ERROR, "Predictor order too large!\n"); + return AVERROR_INVALIDDATA; + } } else { *bd->opt_order = sconf->max_order; } From b6c5848a1f8fc2755ea70d325acaddae9fac45ab Mon Sep 17 00:00:00 2001 From: Mina Nagy Zaki Date: Wed, 8 Jun 2011 19:24:25 +0300 Subject: [PATCH 10/12] lavfi: avfilter_merge_formats: handle case where inputs are same This fixes a double-free crash if lists are the same due to the two merge_ref() calls at the end of the (useless) merging that happens. Signed-off-by: Anton Khirnov (cherry picked from commit 11b6a82412bcd372adf694a26d83b07d337e1325) Conflicts: libavfilter/formats.c Signed-off-by: Reinhard Tartler (cherry picked from commit e5f4e249422834f727bcd432b73af971277f1371) Signed-off-by: Anton Khirnov --- libavfilter/formats.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libavfilter/formats.c b/libavfilter/formats.c index ae916cf16c..1b58b50e9f 100644 --- a/libavfilter/formats.c +++ b/libavfilter/formats.c @@ -44,6 +44,9 @@ AVFilterFormats *avfilter_merge_formats(AVFilterFormats *a, AVFilterFormats *b) AVFilterFormats *ret; unsigned i, j, k = 0; + if (a == b) + return a; + ret = av_mallocz(sizeof(AVFilterFormats)); /* merge list of formats */ From 61ece41372989cca40634a4f32ff6131d19f99a2 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 22 Feb 2012 19:23:18 -0500 Subject: [PATCH 11/12] vorbisenc: check all allocations for failure (cherry picked from commit be8d812c9635f31f69c30dff9ebf565a07a7dab7) Signed-off-by: Anton Khirnov (cherry picked from commit e46cf805b10070327026f8e2880fe29e5e9ac1af) Signed-off-by: Anton Khirnov --- libavcodec/vorbisenc.c | 127 +++++++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 35 deletions(-) diff --git a/libavcodec/vorbisenc.c b/libavcodec/vorbisenc.c index 67d094b37d..ad69c61695 100644 --- a/libavcodec/vorbisenc.c +++ b/libavcodec/vorbisenc.c @@ -155,7 +155,7 @@ static int cb_lookup_vals(int lookup, int dimentions, int entries) return 0; } -static void ready_codebook(vorbis_enc_codebook *cb) +static int ready_codebook(vorbis_enc_codebook *cb) { int i; @@ -167,6 +167,8 @@ static void ready_codebook(vorbis_enc_codebook *cb) int vals = cb_lookup_vals(cb->lookup, cb->ndimentions, cb->nentries); cb->dimentions = av_malloc(sizeof(float) * cb->nentries * cb->ndimentions); cb->pow2 = av_mallocz(sizeof(float) * cb->nentries); + if (!cb->dimentions || !cb->pow2) + return AVERROR(ENOMEM); for (i = 0; i < cb->nentries; i++) { float last = 0; int j; @@ -187,13 +189,16 @@ static void ready_codebook(vorbis_enc_codebook *cb) cb->pow2[i] /= 2.; } } + return 0; } -static void ready_residue(vorbis_enc_residue *rc, vorbis_enc_context *venc) +static int ready_residue(vorbis_enc_residue *rc, vorbis_enc_context *venc) { int i; assert(rc->type == 2); rc->maxes = av_mallocz(sizeof(float[2]) * rc->classifications); + if (!rc->maxes) + return AVERROR(ENOMEM); for (i = 0; i < rc->classifications; i++) { int j; vorbis_enc_codebook * cb; @@ -223,15 +228,16 @@ static void ready_residue(vorbis_enc_residue *rc, vorbis_enc_context *venc) rc->maxes[i][0] += 0.8; rc->maxes[i][1] += 0.8; } + return 0; } -static void create_vorbis_context(vorbis_enc_context *venc, - AVCodecContext *avccontext) +static int create_vorbis_context(vorbis_enc_context *venc, + AVCodecContext *avccontext) { vorbis_enc_floor *fc; vorbis_enc_residue *rc; vorbis_enc_mapping *mc; - int i, book; + int i, book, ret; venc->channels = avccontext->channels; venc->sample_rate = avccontext->sample_rate; @@ -239,6 +245,8 @@ static void create_vorbis_context(vorbis_enc_context *venc, venc->ncodebooks = FF_ARRAY_ELEMS(cvectors); venc->codebooks = av_malloc(sizeof(vorbis_enc_codebook) * venc->ncodebooks); + if (!venc->codebooks) + return AVERROR(ENOMEM); // codebook 0..14 - floor1 book, values 0..255 // codebook 15 residue masterbook @@ -255,27 +263,36 @@ static void create_vorbis_context(vorbis_enc_context *venc, cb->lens = av_malloc(sizeof(uint8_t) * cb->nentries); cb->codewords = av_malloc(sizeof(uint32_t) * cb->nentries); + if (!cb->lens || !cb->codewords) + return AVERROR(ENOMEM); memcpy(cb->lens, cvectors[book].clens, cvectors[book].len); memset(cb->lens + cvectors[book].len, 0, cb->nentries - cvectors[book].len); if (cb->lookup) { vals = cb_lookup_vals(cb->lookup, cb->ndimentions, cb->nentries); cb->quantlist = av_malloc(sizeof(int) * vals); + if (!cb->quantlist) + return AVERROR(ENOMEM); for (i = 0; i < vals; i++) cb->quantlist[i] = cvectors[book].quant[i]; } else { cb->quantlist = NULL; } - ready_codebook(cb); + if ((ret = ready_codebook(cb)) < 0) + return ret; } venc->nfloors = 1; venc->floors = av_malloc(sizeof(vorbis_enc_floor) * venc->nfloors); + if (!venc->floors) + return AVERROR(ENOMEM); // just 1 floor fc = &venc->floors[0]; fc->partitions = NUM_FLOOR_PARTITIONS; fc->partition_to_class = av_malloc(sizeof(int) * fc->partitions); + if (!fc->partition_to_class) + return AVERROR(ENOMEM); fc->nclasses = 0; for (i = 0; i < fc->partitions; i++) { static const int a[] = {0, 1, 2, 2, 3, 3, 4, 4}; @@ -284,6 +301,8 @@ static void create_vorbis_context(vorbis_enc_context *venc, } fc->nclasses++; fc->classes = av_malloc(sizeof(vorbis_enc_floor_class) * fc->nclasses); + if (!fc->classes) + return AVERROR(ENOMEM); for (i = 0; i < fc->nclasses; i++) { vorbis_enc_floor_class * c = &fc->classes[i]; int j, books; @@ -292,6 +311,8 @@ static void create_vorbis_context(vorbis_enc_context *venc, c->masterbook = floor_classes[i].masterbook; books = (1 << c->subclass); c->books = av_malloc(sizeof(int) * books); + if (!c->books) + return AVERROR(ENOMEM); for (j = 0; j < books; j++) c->books[j] = floor_classes[i].nbooks[j]; } @@ -303,6 +324,8 @@ static void create_vorbis_context(vorbis_enc_context *venc, fc->values += fc->classes[fc->partition_to_class[i]].dim; fc->list = av_malloc(sizeof(vorbis_floor1_entry) * fc->values); + if (!fc->list) + return AVERROR(ENOMEM); fc->list[0].x = 0; fc->list[1].x = 1 << fc->rangebits; for (i = 2; i < fc->values; i++) { @@ -317,6 +340,8 @@ static void create_vorbis_context(vorbis_enc_context *venc, venc->nresidues = 1; venc->residues = av_malloc(sizeof(vorbis_enc_residue) * venc->nresidues); + if (!venc->residues) + return AVERROR(ENOMEM); // single residue rc = &venc->residues[0]; @@ -327,6 +352,8 @@ static void create_vorbis_context(vorbis_enc_context *venc, rc->classifications = 10; rc->classbook = 15; rc->books = av_malloc(sizeof(*rc->books) * rc->classifications); + if (!rc->books) + return AVERROR(ENOMEM); { static const int8_t a[10][8] = { { -1, -1, -1, -1, -1, -1, -1, -1, }, @@ -342,19 +369,26 @@ static void create_vorbis_context(vorbis_enc_context *venc, }; memcpy(rc->books, a, sizeof a); } - ready_residue(rc, venc); + if ((ret = ready_residue(rc, venc)) < 0) + return ret; venc->nmappings = 1; venc->mappings = av_malloc(sizeof(vorbis_enc_mapping) * venc->nmappings); + if (!venc->mappings) + return AVERROR(ENOMEM); // single mapping mc = &venc->mappings[0]; mc->submaps = 1; mc->mux = av_malloc(sizeof(int) * venc->channels); + if (!mc->mux) + return AVERROR(ENOMEM); for (i = 0; i < venc->channels; i++) mc->mux[i] = 0; mc->floor = av_malloc(sizeof(int) * mc->submaps); mc->residue = av_malloc(sizeof(int) * mc->submaps); + if (!mc->floor || !mc->residue) + return AVERROR(ENOMEM); for (i = 0; i < mc->submaps; i++) { mc->floor[i] = 0; mc->residue[i] = 0; @@ -362,6 +396,8 @@ static void create_vorbis_context(vorbis_enc_context *venc, mc->coupling_steps = venc->channels == 2 ? 1 : 0; mc->magnitude = av_malloc(sizeof(int) * mc->coupling_steps); mc->angle = av_malloc(sizeof(int) * mc->coupling_steps); + if (!mc->magnitude || !mc->angle) + return AVERROR(ENOMEM); if (mc->coupling_steps) { mc->magnitude[0] = 0; mc->angle[0] = 1; @@ -369,6 +405,8 @@ static void create_vorbis_context(vorbis_enc_context *venc, venc->nmodes = 1; venc->modes = av_malloc(sizeof(vorbis_enc_mode) * venc->nmodes); + if (!venc->modes) + return AVERROR(ENOMEM); // single mode venc->modes[0].blockflag = 0; @@ -379,12 +417,18 @@ static void create_vorbis_context(vorbis_enc_context *venc, venc->samples = av_malloc(sizeof(float) * venc->channels * (1 << venc->log2_blocksize[1])); venc->floor = av_malloc(sizeof(float) * venc->channels * (1 << venc->log2_blocksize[1]) / 2); venc->coeffs = av_malloc(sizeof(float) * venc->channels * (1 << venc->log2_blocksize[1]) / 2); + if (!venc->saved || !venc->samples || !venc->floor || !venc->coeffs) + return AVERROR(ENOMEM); venc->win[0] = ff_vorbis_vwin[venc->log2_blocksize[0] - 6]; venc->win[1] = ff_vorbis_vwin[venc->log2_blocksize[1] - 6]; - ff_mdct_init(&venc->mdct[0], venc->log2_blocksize[0], 0, 1.0); - ff_mdct_init(&venc->mdct[1], venc->log2_blocksize[1], 0, 1.0); + if ((ret = ff_mdct_init(&venc->mdct[0], venc->log2_blocksize[0], 0, 1.0)) < 0) + return ret; + if ((ret = ff_mdct_init(&venc->mdct[1], venc->log2_blocksize[1], 0, 1.0)) < 0) + return ret; + + return 0; } static void put_float(PutBitContext *pb, float f) @@ -647,6 +691,8 @@ static int put_main_header(vorbis_enc_context *venc, uint8_t **out) len = hlens[0] + hlens[1] + hlens[2]; p = *out = av_mallocz(64 + len + len/255); + if (!p) + return AVERROR(ENOMEM); *p++ = 2; p += av_xiphlacing(p, hlens[0]); @@ -952,32 +998,6 @@ static int apply_window_and_mdct(vorbis_enc_context *venc, const signed short *a return 1; } -static av_cold int vorbis_encode_init(AVCodecContext *avccontext) -{ - vorbis_enc_context *venc = avccontext->priv_data; - - if (avccontext->channels != 2) { - av_log(avccontext, AV_LOG_ERROR, "Current Libav Vorbis encoder only supports 2 channels.\n"); - return -1; - } - - create_vorbis_context(venc, avccontext); - - if (avccontext->flags & CODEC_FLAG_QSCALE) - venc->quality = avccontext->global_quality / (float)FF_QP2LAMBDA / 10.; - else - venc->quality = 0.03; - venc->quality *= venc->quality; - - avccontext->extradata_size = put_main_header(venc, (uint8_t**)&avccontext->extradata); - - avccontext->frame_size = 1 << (venc->log2_blocksize[0] - 1); - - avccontext->coded_frame = avcodec_alloc_frame(); - avccontext->coded_frame->key_frame = 1; - - return 0; -} static int vorbis_encode_frame(AVCodecContext *avccontext, unsigned char *packets, @@ -1102,6 +1122,43 @@ static av_cold int vorbis_encode_close(AVCodecContext *avccontext) return 0 ; } +static av_cold int vorbis_encode_init(AVCodecContext *avccontext) +{ + vorbis_enc_context *venc = avccontext->priv_data; + int ret; + + if (avccontext->channels != 2) { + av_log(avccontext, AV_LOG_ERROR, "Current Libav Vorbis encoder only supports 2 channels.\n"); + return -1; + } + + if ((ret = create_vorbis_context(venc, avccontext)) < 0) + goto error; + + if (avccontext->flags & CODEC_FLAG_QSCALE) + venc->quality = avccontext->global_quality / (float)FF_QP2LAMBDA / 10.; + else + venc->quality = 0.03; + venc->quality *= venc->quality; + + if ((ret = put_main_header(venc, (uint8_t**)&avccontext->extradata)) < 0) + goto error; + avccontext->extradata_size = ret; + + avccontext->frame_size = 1 << (venc->log2_blocksize[0] - 1); + + avccontext->coded_frame = avcodec_alloc_frame(); + if (!avccontext->coded_frame) { + ret = AVERROR(ENOMEM); + goto error; + } + + return 0; +error: + vorbis_encode_close(avccontext); + return ret; +} + AVCodec ff_vorbis_encoder = { "vorbis", AVMEDIA_TYPE_AUDIO, From d6e250abfc36b239ef0c1fc9d45d588b853bfcb9 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Mon, 4 Jun 2012 18:27:03 -0700 Subject: [PATCH 12/12] vorbis: Validate that the floor 1 X values contain no duplicates. Duplicate values in this vector are explicitly banned by the Vorbis I spec and cause divide-by-zero crashes later on. (cherry picked from commit ecf79c4d3e8baaf2f303278ef81db6f8407656bc) Signed-off-by: Reinhard Tartler (cherry picked from commit 9aaaeba45c41cf2b3fa4100abbdee7437428f93c) Signed-off-by: Anton Khirnov --- libavcodec/vorbis.c | 9 ++++++++- libavcodec/vorbis.h | 3 ++- libavcodec/vorbisdec.c | 6 +++++- libavcodec/vorbisenc.c | 3 ++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c index 20be707313..e0eda46f71 100644 --- a/libavcodec/vorbis.c +++ b/libavcodec/vorbis.c @@ -117,7 +117,8 @@ int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, unsigned num) return 0; } -void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values) +int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext, + vorbis_floor1_entry *list, int values) { int i; list[0].sort = 0; @@ -141,6 +142,11 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values) for (i = 0; i < values - 1; i++) { int j; for (j = i + 1; j < values; j++) { + if (list[i].x == list[j].x) { + av_log(avccontext, AV_LOG_ERROR, + "Duplicate value found in floor 1 X coordinates\n"); + return AVERROR_INVALIDDATA; + } if (list[list[i].sort].x > list[list[j].sort].x) { int tmp = list[i].sort; list[i].sort = list[j].sort; @@ -148,6 +154,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values) } } } + return 0; } static inline void render_line_unrolled(intptr_t x, int y, int x1, diff --git a/libavcodec/vorbis.h b/libavcodec/vorbis.h index 8501e0a178..b4346fbbe7 100644 --- a/libavcodec/vorbis.h +++ b/libavcodec/vorbis.h @@ -36,7 +36,8 @@ typedef struct { uint16_t high; } vorbis_floor1_entry; -void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values); +int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext, + vorbis_floor1_entry *list, int values); unsigned int ff_vorbis_nth_root(unsigned int x, unsigned int n); // x^(1/n) int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, unsigned num); void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values, diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c index d6850b7f48..3805d27627 100644 --- a/libavcodec/vorbisdec.c +++ b/libavcodec/vorbisdec.c @@ -559,7 +559,11 @@ static int vorbis_parse_setup_hdr_floors(vorbis_context *vc) } // Precalculate order of x coordinates - needed for decode - ff_vorbis_ready_floor1_list(floor_setup->data.t1.list, floor_setup->data.t1.x_list_dim); + if (ff_vorbis_ready_floor1_list(vc->avccontext, + floor_setup->data.t1.list, + floor_setup->data.t1.x_list_dim)) { + return AVERROR_INVALIDDATA; + } } else if (floor_setup->floor_type == 0) { unsigned max_codebook_dim = 0; diff --git a/libavcodec/vorbisenc.c b/libavcodec/vorbisenc.c index ad69c61695..7311c05d36 100644 --- a/libavcodec/vorbisenc.c +++ b/libavcodec/vorbisenc.c @@ -336,7 +336,8 @@ static int create_vorbis_context(vorbis_enc_context *venc, }; fc->list[i].x = a[i - 2]; } - ff_vorbis_ready_floor1_list(fc->list, fc->values); + if (ff_vorbis_ready_floor1_list(avccontext, fc->list, fc->values)) + return AVERROR(EINVAL); venc->nresidues = 1; venc->residues = av_malloc(sizeof(vorbis_enc_residue) * venc->nresidues);