diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index fead5ae116..4fc2aed302 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1587,7 +1587,9 @@ again:
 #endif
                     } else
                         context_count++;
-                }
+                } else
+                    sl->ref_count[0] = sl->ref_count[1] = 0;
+            break;
                 break;
             case NAL_DPA:
             case NAL_DPB: