From 562f2ba4ed07baed407ccd314c8358446caa5e1e Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 23 Aug 2016 11:00:29 +0200 Subject: [PATCH] avcodec/aacenc: Tighter input checks Fixes occurance of NaN/Inf leading to assertion failures and out of array access Fixes: d1c38a09acc34845c6be3a127a5aacaf/signal_sigsegv_3982225_6121_d18bd5451d4245ee09408f04badd1b83.wmv Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer (cherry picked from commit 77bf96b04710b98a52aaddb93bfd32da0d506191) Signed-off-by: Michael Niedermayer --- libavcodec/aacenc.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/libavcodec/aacenc.c b/libavcodec/aacenc.c index 7b643f51f7..86d40855e7 100644 --- a/libavcodec/aacenc.c +++ b/libavcodec/aacenc.c @@ -493,6 +493,7 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt, int ms_mode = 0, is_mode = 0, tns_mode = 0, pred_mode = 0; int chan_el_counter[4]; FFPsyWindowInfo windows[AAC_MAX_CHANNELS]; + int k; if (s->last_frame == 2) return 0; @@ -573,16 +574,11 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt, apply_window_and_mdct(s, &cpe->ch[ch], overlap); - if (isnan(cpe->ch[ch].coeffs[ 0]) || isinf(cpe->ch[ch].coeffs[ 0]) || - isnan(cpe->ch[ch].coeffs[ 128]) || isinf(cpe->ch[ch].coeffs[ 128]) || - isnan(cpe->ch[ch].coeffs[2*128]) || isinf(cpe->ch[ch].coeffs[2*128]) || - isnan(cpe->ch[ch].coeffs[3*128]) || isinf(cpe->ch[ch].coeffs[3*128]) || - isnan(cpe->ch[ch].coeffs[4*128]) || isinf(cpe->ch[ch].coeffs[4*128]) || - isnan(cpe->ch[ch].coeffs[5*128]) || isinf(cpe->ch[ch].coeffs[5*128]) || - isnan(cpe->ch[ch].coeffs[6*128]) || isinf(cpe->ch[ch].coeffs[6*128]) || - isnan(cpe->ch[ch].coeffs[7*128]) || isinf(cpe->ch[ch].coeffs[7*128])) { - av_log(avctx, AV_LOG_ERROR, "Input contains NaN/+-Inf\n"); - return AVERROR(EINVAL); + for (k = 0; k < 1024; k++) { + if (!(fabs(cpe->ch[ch].coeffs[k]) < 1E16)) { // Ensure headroom for energy calculation + av_log(avctx, AV_LOG_ERROR, "Input contains (near) NaN/+-Inf\n"); + return AVERROR(EINVAL); + } } avoid_clipping(s, &cpe->ch[ch]); }