From ee6ef66d65730f426d25e7c942b7db4a3cb4bb1a Mon Sep 17 00:00:00 2001 From: Paul B Mahol Date: Fri, 12 May 2023 21:01:41 +0200 Subject: [PATCH] avfilter/avfiltergraph: fix check for negative return Before this commit if allocation would fail in ff_add_channel_layout() function, function would return negative error code and this would cause wrong format pick up later. If allocation would not fail return code would be 0 and then format negotiation would simply fail as code would break from the loop but with wrong return code. Error was introduced in 6aaac24d72a7da commit. Fixes #6638 --- libavfilter/avfiltergraph.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c index 53f468494d..744f480e1d 100644 --- a/libavfilter/avfiltergraph.c +++ b/libavfilter/avfiltergraph.c @@ -748,8 +748,10 @@ static int reduce_formats_on_filter(AVFilterContext *filter) (KNOWN(fmt) || fmts->all_counts)) { /* Turn the infinite list into a singleton */ fmts->all_layouts = fmts->all_counts = 0; - if (ff_add_channel_layout(&outlink->incfg.channel_layouts, fmt) < 0) - ret = 1; + ret = ff_add_channel_layout(&outlink->incfg.channel_layouts, fmt); + if (ret < 0) + return ret; + ret = 1; break; }