From d391719be19be2f2716dcb1da9f88b0b4214e4c4 Mon Sep 17 00:00:00 2001 From: Andreas Cadhalpun Date: Thu, 13 Oct 2016 22:14:46 +0200 Subject: [PATCH] libopenjpegenc: fix out-of-bounds reads when filling the edges The calculation of width/height should round up, not round down to prevent setting width or height to 0. Also image->comps[compno].w is unsigned (at least in openjpeg2), so the calculation could silently wrap around without the explicit cast to int. Reviewed-by: Michael Bradshaw Reviewed-by: Michael Niedermayer Signed-off-by: Andreas Cadhalpun (cherry picked from commit 56706ac0d5723cb549fec2602e798ab1bf6004cd) Signed-off-by: Andreas Cadhalpun --- libavcodec/libopenjpegenc.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libavcodec/libopenjpegenc.c b/libavcodec/libopenjpegenc.c index 857ee1adf9..1b7e1684af 100644 --- a/libavcodec/libopenjpegenc.c +++ b/libavcodec/libopenjpegenc.c @@ -421,7 +421,7 @@ static int libopenjpeg_copy_packed8(AVCodecContext *avctx, const AVFrame *frame, for (; y < image->comps[compno].h; ++y) { image_line = image->comps[compno].data + y * image->comps[compno].w; for (x = 0; x < image->comps[compno].w; ++x) { - image_line[x] = image_line[x - image->comps[compno].w]; + image_line[x] = image_line[x - (int)image->comps[compno].w]; } } } @@ -461,7 +461,7 @@ static int libopenjpeg_copy_packed12(AVCodecContext *avctx, const AVFrame *frame for (; y < image->comps[compno].h; ++y) { image_line = image->comps[compno].data + y * image->comps[compno].w; for (x = 0; x < image->comps[compno].w; ++x) { - image_line[x] = image_line[x - image->comps[compno].w]; + image_line[x] = image_line[x - (int)image->comps[compno].w]; } } } @@ -501,7 +501,7 @@ static int libopenjpeg_copy_packed16(AVCodecContext *avctx, const AVFrame *frame for (; y < image->comps[compno].h; ++y) { image_line = image->comps[compno].data + y * image->comps[compno].w; for (x = 0; x < image->comps[compno].w; ++x) { - image_line[x] = image_line[x - image->comps[compno].w]; + image_line[x] = image_line[x - (int)image->comps[compno].w]; } } } @@ -528,8 +528,8 @@ static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *fram } for (compno = 0; compno < numcomps; ++compno) { - width = avctx->width / image->comps[compno].dx; - height = avctx->height / image->comps[compno].dy; + width = (avctx->width + image->comps[compno].dx - 1) / image->comps[compno].dx; + height = (avctx->height + image->comps[compno].dy - 1) / image->comps[compno].dy; for (y = 0; y < height; ++y) { image_line = image->comps[compno].data + y * image->comps[compno].w; frame_index = y * frame->linesize[compno]; @@ -542,7 +542,7 @@ static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *fram for (; y < image->comps[compno].h; ++y) { image_line = image->comps[compno].data + y * image->comps[compno].w; for (x = 0; x < image->comps[compno].w; ++x) { - image_line[x] = image_line[x - image->comps[compno].w]; + image_line[x] = image_line[x - (int)image->comps[compno].w]; } } } @@ -570,8 +570,8 @@ static int libopenjpeg_copy_unpacked16(AVCodecContext *avctx, const AVFrame *fra } for (compno = 0; compno < numcomps; ++compno) { - width = avctx->width / image->comps[compno].dx; - height = avctx->height / image->comps[compno].dy; + width = (avctx->width + image->comps[compno].dx - 1) / image->comps[compno].dx; + height = (avctx->height + image->comps[compno].dy - 1) / image->comps[compno].dy; frame_ptr = (uint16_t *)frame->data[compno]; for (y = 0; y < height; ++y) { image_line = image->comps[compno].data + y * image->comps[compno].w; @@ -585,7 +585,7 @@ static int libopenjpeg_copy_unpacked16(AVCodecContext *avctx, const AVFrame *fra for (; y < image->comps[compno].h; ++y) { image_line = image->comps[compno].data + y * image->comps[compno].w; for (x = 0; x < image->comps[compno].w; ++x) { - image_line[x] = image_line[x - image->comps[compno].w]; + image_line[x] = image_line[x - (int)image->comps[compno].w]; } } }