From 02327d1237f19619272e7b04b715e807c16af6a5 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Fri, 19 May 2017 11:47:21 +0200 Subject: [PATCH] decode: fix the code reducing cropping to preserve alignment Currently it does not work at all. Libav Bug-Id: 1058 (cherry picked from commit 8652a2c24836ce5546b398f12b7fed45000050e1) --- libavcodec/decode.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libavcodec/decode.c b/libavcodec/decode.c index fabdab3694..e423e68eab 100644 --- a/libavcodec/decode.c +++ b/libavcodec/decode.c @@ -763,6 +763,7 @@ static int apply_cropping(AVCodecContext *avctx, AVFrame *frame) /* adjust the offsets to avoid breaking alignment */ if (!(avctx->flags & AV_CODEC_FLAG_UNALIGNED)) { + int log2_crop_align = frame->crop_left ? av_ctz(frame->crop_left) : INT_MAX; int min_log2_align = INT_MAX; for (i = 0; frame->data[i]; i++) { @@ -770,8 +771,13 @@ static int apply_cropping(AVCodecContext *avctx, AVFrame *frame) min_log2_align = FFMIN(log2_align, min_log2_align); } + /* we assume, and it should always be true, that the data alignment is + * related to the cropping alignment by a constant power-of-2 factor */ + if (log2_crop_align < min_log2_align) + return AVERROR_BUG; + if (min_log2_align < 5) { - frame->crop_left &= ~((1 << min_log2_align) - 1); + frame->crop_left &= ~((1 << (5 + log2_crop_align - min_log2_align)) - 1); calc_cropping_offsets(offsets, frame, desc); } }