diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c
index 3e17e0fcac..b038efed9c 100644
--- a/libavcodec/v4l2_m2m_dec.c
+++ b/libavcodec/v4l2_m2m_dec.c
@@ -138,14 +138,10 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
     V4L2m2mContext *s = ((V4L2m2mPriv*)avctx->priv_data)->context;
     V4L2Context *const capture = &s->capture;
     V4L2Context *const output = &s->output;
-    AVPacket avpkt = {0};
     int ret;
 
-    if (s->buf_pkt.size) {
-        avpkt = s->buf_pkt;
-        memset(&s->buf_pkt, 0, sizeof(AVPacket));
-    } else {
-        ret = ff_decode_get_packet(avctx, &avpkt);
+    if (!s->buf_pkt.size) {
+        ret = ff_decode_get_packet(avctx, &s->buf_pkt);
         if (ret < 0 && ret != AVERROR_EOF)
             return ret;
     }
@@ -153,32 +149,29 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
     if (s->draining)
         goto dequeue;
 
-    ret = ff_v4l2_context_enqueue_packet(output, &avpkt);
-    if (ret < 0) {
-        if (ret != AVERROR(EAGAIN))
-           return ret;
+    ret = ff_v4l2_context_enqueue_packet(output, &s->buf_pkt);
+    if (ret < 0 && ret != AVERROR(EAGAIN))
+        goto fail;
 
-        s->buf_pkt = avpkt;
-        /* no input buffers available, continue dequeing */
-    }
+    /* if EAGAIN don't unref packet and try to enqueue in the next iteration */
+    if (ret != AVERROR(EAGAIN))
+        av_packet_unref(&s->buf_pkt);
 
-    if (avpkt.size) {
+    if (!s->draining) {
         ret = v4l2_try_start(avctx);
         if (ret) {
-            av_packet_unref(&avpkt);
-
             /* cant recover */
-            if (ret == AVERROR(ENOMEM))
-                return ret;
-
-            return 0;
+            if (ret != AVERROR(ENOMEM))
+                ret = 0;
+            goto fail;
         }
     }
 
 dequeue:
-    if (!s->buf_pkt.size)
-        av_packet_unref(&avpkt);
     return ff_v4l2_context_dequeue_frame(capture, frame, -1);
+fail:
+    av_packet_unref(&s->buf_pkt);
+    return ret;
 }
 
 static av_cold int v4l2_decode_init(AVCodecContext *avctx)