diff --git a/libavcodec/v4l2_m2m.c b/libavcodec/v4l2_m2m.c
index 31600617fd..cdfd579810 100644
--- a/libavcodec/v4l2_m2m.c
+++ b/libavcodec/v4l2_m2m.c
@@ -329,6 +329,7 @@ static void v4l2_m2m_destroy_context(void *opaque, uint8_t *context)
     sem_destroy(&s->refsync);
 
     close(s->fd);
+    av_frame_unref(s->frame);
     av_frame_free(&s->frame);
     av_packet_unref(&s->buf_pkt);
 
diff --git a/libavcodec/v4l2_m2m_enc.c b/libavcodec/v4l2_m2m_enc.c
index af0ed1e306..4230a415fd 100644
--- a/libavcodec/v4l2_m2m_enc.c
+++ b/libavcodec/v4l2_m2m_enc.c
@@ -295,16 +295,20 @@ static int v4l2_receive_packet(AVCodecContext *avctx, AVPacket *avpkt)
     if (s->draining)
         goto dequeue;
 
+    if (!frame->buf[0]) {
     ret = ff_encode_get_frame(avctx, frame);
     if (ret < 0 && ret != AVERROR_EOF)
         return ret;
 
     if (ret == AVERROR_EOF)
         frame = NULL;
+    }
 
     ret = v4l2_send_frame(avctx, frame);
-    av_frame_unref(frame);
-    if (ret < 0)
+    if (ret != AVERROR(EAGAIN))
+        av_frame_unref(frame);
+
+    if (ret < 0 && ret != AVERROR(EAGAIN))
         return ret;
 
     if (!output->streamon) {