diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 2cdb67f978..e43ce428c5 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -373,9 +373,11 @@ int ff_h264_build_ref_list(H264Context *h, H264SliceContext *sl)
                 av_assert0(0);
             }
 
-            if (i < 0) {
+            if (i < 0 || mismatches_ref(h, ref)) {
                 av_log(h->avctx, AV_LOG_ERROR,
-                       "reference picture missing during reorder\n");
+                       i < 0 ? "reference picture missing during reorder\n" :
+                               "mismatching reference\n"
+                      );
                 memset(&sl->ref_list[list][index], 0, sizeof(sl->ref_list[0][0])); // FIXME
             } else {
                 for (i = index; i + 1 < sl->ref_count[list]; i++) {