highperfocused/ffmpeg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

avcodec/dnxhdenc: retry increasing qscale to not overflow max_bits

Browse Source 
Increase mb_bits type from uint16_t to uint32_t to fix possible overflows
in bit size calculations.

Update fate test that needs change.
This commit is contained in:
Paul B Mahol
2022-03-05 11:10:24 +01:00
parent 37480b1b85
commit 044c09c0a0
3 changed files with 8 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
libavcodec/dnxhdenc.c
View File

@@ -924,7 +924,7 @@ static void dnxhd_setup_threads_slices(DNXHDEncContext *ctx)
unsigned mb = mb_y * ctx->m.mb_width + mb_x; unsigned mb = mb_y * ctx->m.mb_width + mb_x;
ctx->slice_size[mb_y] += ctx->mb_bits[mb]; ctx->slice_size[mb_y] += ctx->mb_bits[mb];
} }
ctx->slice_size[mb_y] = (ctx->slice_size[mb_y] + 31) & ~31; ctx->slice_size[mb_y] = (ctx->slice_size[mb_y] + 31U) & ~31U;
ctx->slice_size[mb_y] >>= 3; ctx->slice_size[mb_y] >>= 3;
thread_size = ctx->slice_size[mb_y]; thread_size = ctx->slice_size[mb_y];
offset += thread_size; offset += thread_size;
@@ -1220,17 +1220,19 @@ static int dnxhd_encode_fast(AVCodecContext *avctx, DNXHDEncContext *ctx)
avctx->execute2(avctx, dnxhd_mb_var_thread, avctx->execute2(avctx, dnxhd_mb_var_thread,
NULL, NULL, ctx->m.mb_height); NULL, NULL, ctx->m.mb_height);
radix_sort(ctx->mb_cmp, ctx->mb_cmp_tmp, ctx->m.mb_num); radix_sort(ctx->mb_cmp, ctx->mb_cmp_tmp, ctx->m.mb_num);
retry:
for (x = 0; x < ctx->m.mb_num && max_bits > ctx->frame_bits; x++) { for (x = 0; x < ctx->m.mb_num && max_bits > ctx->frame_bits; x++) {
int mb = ctx->mb_cmp[x].mb; int mb = ctx->mb_cmp[x].mb;
int rc = (ctx->qscale * ctx->m.mb_num ) + mb; int rc = (ctx->qscale * ctx->m.mb_num ) + mb;
max_bits -= ctx->mb_rc[rc].bits - max_bits -= ctx->mb_rc[rc].bits -
ctx->mb_rc[rc + ctx->m.mb_num].bits; ctx->mb_rc[rc + ctx->m.mb_num].bits;
ctx->mb_qscale[mb] = ctx->qscale + 1; if (ctx->mb_qscale[mb] < 255)
ctx->mb_qscale[mb]++;
ctx->mb_bits[mb] = ctx->mb_rc[rc + ctx->m.mb_num].bits; ctx->mb_bits[mb] = ctx->mb_rc[rc + ctx->m.mb_num].bits;
} }
if (max_bits > ctx->frame_bits) if (max_bits > ctx->frame_bits)
return AVERROR(EINVAL); goto retry;
} }
return 0; return 0;
} }

2
libavcodec/dnxhdenc.h
View File

@@ -100,7 +100,7 @@ typedef struct DNXHDEncContext {
unsigned qscale; unsigned qscale;
unsigned lambda; unsigned lambda;
uint16_t *mb_bits; uint32_t *mb_bits;
uint8_t *mb_qscale; uint8_t *mb_qscale;
RCCMPEntry *mb_cmp; RCCMPEntry *mb_cmp;

4
tests/ref/lavf/mxf_opatom
View File

@@ -1,3 +1,3 @@
aab6397829bd90f0c77a3f9fde53bb9c *tests/data/lavf/lavf.mxf_opatom b1e32792b835ac51b0b1d5dcb9497e4e *tests/data/lavf/lavf.mxf_opatom
4717625 tests/data/lavf/lavf.mxf_opatom 4717625 tests/data/lavf/lavf.mxf_opatom
tests/data/lavf/lavf.mxf_opatom CRC=0xf55aa22a tests/data/lavf/lavf.mxf_opatom CRC=0xb13ba2f8