diff --git a/libavcodec/cbs_av1.c b/libavcodec/cbs_av1.c
index eb6b801790..533a4ffa16 100644
--- a/libavcodec/cbs_av1.c
+++ b/libavcodec/cbs_av1.c
@@ -940,6 +940,8 @@ static int cbs_av1_read_unit(CodedBitstreamContext *ctx,
         priv->spatial_id  = 0;
     }
 
+    priv->ref = (AV1ReferenceFrameState *)&priv->read_ref;
+
     switch (obu->header.obu_type) {
     case AV1_OBU_SEQUENCE_HEADER:
         {
@@ -1082,6 +1084,8 @@ static int cbs_av1_write_obu(CodedBitstreamContext *ctx,
     td = NULL;
     start_pos = put_bits_count(pbc);
 
+    priv->ref = (AV1ReferenceFrameState *)&priv->write_ref;
+
     switch (obu->header.obu_type) {
     case AV1_OBU_SEQUENCE_HEADER:
         {
diff --git a/libavcodec/cbs_av1.h b/libavcodec/cbs_av1.h
index 1fb668ada4..675ed28f60 100644
--- a/libavcodec/cbs_av1.h
+++ b/libavcodec/cbs_av1.h
@@ -441,7 +441,9 @@ typedef struct CodedBitstreamAV1Context {
     int tile_cols;
     int tile_rows;
 
-    AV1ReferenceFrameState ref[AV1_NUM_REF_FRAMES];
+    AV1ReferenceFrameState *ref;
+    AV1ReferenceFrameState read_ref[AV1_NUM_REF_FRAMES];
+    AV1ReferenceFrameState write_ref[AV1_NUM_REF_FRAMES];
 
     // Write buffer.
     uint8_t *write_buffer;