diff --git a/libavcodec/cbs_sei.c b/libavcodec/cbs_sei.c
index 2a96db9674..141e97ec58 100644
--- a/libavcodec/cbs_sei.c
+++ b/libavcodec/cbs_sei.c
@@ -262,14 +262,6 @@ int ff_cbs_sei_add_message(CodedBitstreamContext *ctx,
     if (!desc)
         return AVERROR(EINVAL);
 
-    if (payload_buf) {
-        payload_ref = av_buffer_ref(payload_buf);
-        if (!payload_ref)
-            return AVERROR(ENOMEM);
-    } else {
-        payload_ref = NULL;
-    }
-
     // Find an existing SEI unit or make a new one to add to.
     err = cbs_sei_get_unit(ctx, au, prefix, &unit);
     if (err < 0)
@@ -285,6 +277,14 @@ int ff_cbs_sei_add_message(CodedBitstreamContext *ctx,
     if (err < 0)
         return err;
 
+    if (payload_buf) {
+        payload_ref = av_buffer_ref(payload_buf);
+        if (!payload_ref)
+            return AVERROR(ENOMEM);
+    } else {
+        payload_ref = NULL;
+    }
+
     message = &list->messages[list->nb_messages - 1];
 
     message->payload_type = payload_type;