diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index bba77d15b0..88aaac7e1b 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -621,6 +621,8 @@ int ff_h264_execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count)
                      * Report the problem and keep the pair where it is,
                      * and mark this field valid.
                      */
+            if (h->short_ref[0] == h->cur_pic_ptr)
+                remove_short_at_index(h, 0);
 
             if (h->long_ref[mmco[i].long_arg] != h->cur_pic_ptr) {
                 remove_long(h, mmco[i].long_arg, 0);