diff --git a/libavformat/concatdec.c b/libavformat/concatdec.c
index d21805fe71..d226e1504e 100644
--- a/libavformat/concatdec.c
+++ b/libavformat/concatdec.c
@@ -555,9 +555,7 @@ static int concat_read_packet(AVFormatContext *avf, AVPacket *pkt)
 
     while (1) {
         ret = av_read_frame(cat->avf, pkt);
-        if (ret == AVERROR_EOF || packet_after_outpoint(cat, pkt)) {
-            if (ret == 0)
-                av_packet_unref(pkt);
+        if (ret == AVERROR_EOF) {
             if ((ret = open_next_file(avf)) < 0)
                 return ret;
             continue;
@@ -568,6 +566,12 @@ static int concat_read_packet(AVFormatContext *avf, AVPacket *pkt)
             av_packet_unref(pkt);
             return ret;
         }
+        if (packet_after_outpoint(cat, pkt)) {
+            av_packet_unref(pkt);
+            if ((ret = open_next_file(avf)) < 0)
+                return ret;
+            continue;
+        }
         cs = &cat->cur_file->streams[pkt->stream_index];
         if (cs->out_stream_index < 0) {
             av_packet_unref(pkt);