diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 34c51c23d9..0165be8f77 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -1415,7 +1415,9 @@ static int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *fr
     AVFrameSideData *sd;
     int ret;
 
-    av_buffer_unref(&ifp->hw_frames_ctx);
+    ret = av_buffer_replace(&ifp->hw_frames_ctx, frame->hw_frames_ctx);
+    if (ret < 0)
+        return ret;
 
     ifilter->format = frame->format;
 
@@ -1433,12 +1435,6 @@ static int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *fr
     if (sd)
         ifp->displaymatrix = av_memdup(sd->data, sizeof(int32_t) * 9);
 
-    if (frame->hw_frames_ctx) {
-        ifp->hw_frames_ctx = av_buffer_ref(frame->hw_frames_ctx);
-        if (!ifp->hw_frames_ctx)
-            return AVERROR(ENOMEM);
-    }
-
     return 0;
 }