From e028020213cdfd2d37934fa12e11c586040aca4e Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Wed, 18 May 2022 02:10:52 +0200 Subject: [PATCH] avfilter/vf_signature: Fix integer overflow in filter_frame() Fixes: CID1403233 The second of the 2 changes may be unneeded but will help coverity Signed-off-by: Michael Niedermayer (cherry picked from commit dd6040675ec18d19429f882caea6bb306ed6677a) Signed-off-by: Michael Niedermayer --- libavfilter/vf_signature.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavfilter/vf_signature.c b/libavfilter/vf_signature.c index 4ca57ebf1d..66149dcc01 100644 --- a/libavfilter/vf_signature.c +++ b/libavfilter/vf_signature.c @@ -219,7 +219,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *picref) dw1 = inlink->w / 32; if (inlink->w % 32) dw2 = dw1 + 1; - denom = (sc->divide) ? dh1 * dh2 * dw1 * dw2 : 1; + denom = (sc->divide) ? dh1 * (int64_t)dh2 * dw1 * dw2 : 1; for (i = 0; i < 32; i++) { rowcount = 0; @@ -245,7 +245,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *picref) } } - denom = (sc->divide) ? 1 : dh1 * dh2 * dw1 * dw2; + denom = (sc->divide) ? 1 : dh1 * (int64_t)dh2 * dw1 * dw2; for (i = 0; i < ELEMENT_COUNT; i++) { const ElemCat* elemcat = elements[i];