diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index 2f49d0ae6f..2d93c2ab60 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -209,6 +209,7 @@ static void pred_spatial_direct_motion(H264Context * const h, int *mb_type){
                     mv[list]= AV_RN32A(C);
                 }
             }
+            av_assert2(ref[list] < (h->ref_count[list] << !!FRAME_MBAFF(h)));
         }else{
             int mask= ~(MB_TYPE_L0 << (2*list));
             mv[list] = 0;