From 0a6324fe9315e2a7dc5a5fe377f54ff17056f74c Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 24 Dec 2015 21:46:15 +0100 Subject: [PATCH] swscale/utils: Fix intermediate format for cascaded alpha downscaling Fixes Ticket4926 Signed-off-by: Michael Niedermayer (cherry picked from commit b83d8be6bff7d645469a623aee0b380541da15cf) Signed-off-by: Michael Niedermayer --- libswscale/utils.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libswscale/utils.c b/libswscale/utils.c index 2a88dc96f7..5610701136 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -1786,6 +1786,9 @@ fail: // FIXME replace things by appropriate error codes int tmpH = sqrt(srcH * (int64_t)dstH); enum AVPixelFormat tmpFormat = AV_PIX_FMT_YUV420P; + if (isALPHA(srcFormat)) + tmpFormat = AV_PIX_FMT_YUVA420P; + if (srcW*(int64_t)srcH <= 4LL*dstW*dstH) return AVERROR(EINVAL);