diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c
index 81dd6535c0..98dabfc907 100644
--- a/libavcodec/vp3.c
+++ b/libavcodec/vp3.c
@@ -2498,19 +2498,14 @@ static int update_frames(AVCodecContext *avctx)
     Vp3DecodeContext *s = avctx->priv_data;
     int ret = 0;
 
-    /* shuffle frames (last = current) */
-    ff_thread_release_ext_buffer(avctx, &s->last_frame);
-    ret = ff_thread_ref_frame(&s->last_frame, &s->current_frame);
-    if (ret < 0)
-        goto fail;
-
     if (s->keyframe) {
         ff_thread_release_ext_buffer(avctx, &s->golden_frame);
         ret = ff_thread_ref_frame(&s->golden_frame, &s->current_frame);
     }
+    /* shuffle frames */
+    ff_thread_release_ext_buffer(avctx, &s->last_frame);
+    FFSWAP(ThreadFrame, s->last_frame, s->current_frame);
 
-fail:
-    ff_thread_release_ext_buffer(avctx, &s->current_frame);
     return ret;
 }