diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index df4e30524d..a6a1a653ca 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -1463,9 +1463,7 @@ int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
                 frame->extended_buf    = NULL;
                 frame->nb_extended_buf = 0;
             }
-        }
-
-        if (ret < 0 && frame->data[0])
+        } else if (frame->data[0])
             av_frame_unref(frame);
     }