diff --git a/libavformat/rtspdec.c b/libavformat/rtspdec.c
index 9306384555..063e82501c 100644
--- a/libavformat/rtspdec.c
+++ b/libavformat/rtspdec.c
@@ -159,8 +159,9 @@ static int rtsp_read_header(AVFormatContext *s)
     if (ret)
         return ret;
 
-    rt->real_setup_cache = av_mallocz(2 * s->nb_streams * sizeof(*rt->real_setup_cache));
-    if (!rt->real_setup_cache)
+    rt->real_setup_cache = !s->nb_streams ? NULL :
+                           av_mallocz(2 * s->nb_streams * sizeof(*rt->real_setup_cache));
+    if (!rt->real_setup_cache && s->nb_streams)
         return AVERROR(ENOMEM);
     rt->real_setup = rt->real_setup_cache + s->nb_streams;