diff --git a/libavcodec/cbs_av1.c b/libavcodec/cbs_av1.c
index 80628c72ba..c680b34f17 100644
--- a/libavcodec/cbs_av1.c
+++ b/libavcodec/cbs_av1.c
@@ -978,6 +978,8 @@ static int cbs_av1_read_unit(CodedBitstreamContext *ctx,
         priv->spatial_id  = 0;
     }
 
+    priv->ref = (AV1ReferenceFrameState *)&priv->read_ref;
+
     switch (obu->header.obu_type) {
     case AV1_OBU_SEQUENCE_HEADER:
         {
@@ -1114,6 +1116,8 @@ static int cbs_av1_write_obu(CodedBitstreamContext *ctx,
     td = NULL;
     start_pos = put_bits_count(pbc);
 
+    priv->ref = (AV1ReferenceFrameState *)&priv->write_ref;
+
     switch (obu->header.obu_type) {
     case AV1_OBU_SEQUENCE_HEADER:
         {
diff --git a/libavcodec/cbs_av1.h b/libavcodec/cbs_av1.h
index 0546623bea..144d1ebbf7 100644
--- a/libavcodec/cbs_av1.h
+++ b/libavcodec/cbs_av1.h
@@ -421,7 +421,9 @@ typedef struct CodedBitstreamAV1Context {
     int tile_cols;
     int tile_rows;
 
-    AV1ReferenceFrameState ref[AV1_NUM_REF_FRAMES];
+    AV1ReferenceFrameState *ref;
+    AV1ReferenceFrameState read_ref[AV1_NUM_REF_FRAMES];
+    AV1ReferenceFrameState write_ref[AV1_NUM_REF_FRAMES];
 
     // Write buffer.
     uint8_t *write_buffer;