diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index e024a32321..8635aec949 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -748,10 +748,6 @@ int ff_decode_get_hw_frames_ctx(AVCodecContext *avctx,
         // We guarantee 4 base work surfaces. The function above guarantees 1
         // (the absolute minimum), so add the missing count.
         frames_ctx->initial_pool_size += 3;
-
-        // Add an additional surface per thread is frame threading is enabled.
-        if (avctx->active_thread_type & FF_THREAD_FRAME)
-            frames_ctx->initial_pool_size += avctx->thread_count;
     }
 
     ret = av_hwframe_ctx_init(avctx->hw_frames_ctx);
@@ -798,6 +794,11 @@ int avcodec_get_hw_frames_parameters(AVCodecContext *avctx,
             // available then add them here.
             if (avctx->extra_hw_frames > 0)
                 frames_ctx->initial_pool_size += avctx->extra_hw_frames;
+
+            // If frame threading is enabled then an extra surface per thread
+            // is also required.
+            if (avctx->active_thread_type & FF_THREAD_FRAME)
+                frames_ctx->initial_pool_size += avctx->thread_count;
         }
 
         *out_frames_ref = frames_ref;
diff --git a/libavcodec/dxva2.c b/libavcodec/dxva2.c
index e34409d44a..939b45a65b 100644
--- a/libavcodec/dxva2.c
+++ b/libavcodec/dxva2.c
@@ -608,10 +608,6 @@ int ff_dxva2_common_frame_params(AVCodecContext *avctx,
     else
         num_surfaces += 2;
 
-    /* add extra surfaces for frame threading */
-    if (avctx->active_thread_type & FF_THREAD_FRAME)
-        num_surfaces += avctx->thread_count;
-
     frames_ctx->sw_format = avctx->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ?
                             AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
     frames_ctx->width = FFALIGN(avctx->coded_width, surface_alignment);