diff --git a/libavutil/dict.c b/libavutil/dict.c
index a4f638a1fc..d127bb90aa 100644
--- a/libavutil/dict.c
+++ b/libavutil/dict.c
@@ -126,12 +126,12 @@ int av_dict_set(AVDictionary **pm, const char *key, const char *value,
         }
         m->count++;
     } else {
+        if (!m->count) {
+            av_freep(&m->elems);
+            av_freep(pm);
+        }
         av_freep(&copy_key);
     }
-    if (!m->count) {
-        av_freep(&m->elems);
-        av_freep(pm);
-    }
 
     return 0;