diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index 34a4ff727f..0b19353656 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -89,7 +89,8 @@ static void fill_colmap(H264Context *h, int map[2][16+32], int list, int field,
             for(j=start; j<end; j++){
                 if(4*h->ref_list[0][j].frame_num + (h->ref_list[0][j].reference&3) == poc){
                     int cur_ref= mbafi ? (j-16)^field : j;
-                    map[list][2*old_ref + (rfield^field) + 16] = cur_ref;
+                    if(ref1->mbaff)
+                        map[list][2*old_ref + (rfield^field) + 16] = cur_ref;
                     if(rfield == field || !interl)
                         map[list][old_ref] = cur_ref;
                     break;