diff --git a/libavformat/rtsp.c b/libavformat/rtsp.c
index ce5039ac87..77f6eb2501 100644
--- a/libavformat/rtsp.c
+++ b/libavformat/rtsp.c
@@ -1537,13 +1537,6 @@ int ff_rtsp_make_setup_request(AVFormatContext *s, const char *host, int port,
                 err = AVERROR_INVALIDDATA;
                 goto fail;
             }
-            /* Try to initialize the connection state in a
-             * potential NAT router by sending dummy packets.
-             * RTP/RTCP dummy packets are used for RDT, too.
-             */
-            if (CONFIG_RTPDEC &&
-                !(rt->server_type == RTSP_SERVER_WMS && i > 1) && s->iformat)
-                ff_rtp_send_punch_packets(rtsp_st->rtp_handle);
             break;
         }
         case RTSP_LOWER_TRANSPORT_UDP_MULTICAST: {
diff --git a/libavformat/rtspdec.c b/libavformat/rtspdec.c
index 418f3837b8..659c768423 100644
--- a/libavformat/rtspdec.c
+++ b/libavformat/rtspdec.c
@@ -504,6 +504,18 @@ static int rtsp_read_play(AVFormatContext *s)
     av_log(s, AV_LOG_DEBUG, "hello state=%d\n", rt->state);
     rt->nb_byes = 0;
 
+    if (rt->lower_transport == RTSP_LOWER_TRANSPORT_UDP) {
+        for (i = 0; i < rt->nb_rtsp_streams; i++) {
+            RTSPStream *rtsp_st = rt->rtsp_streams[i];
+            /* Try to initialize the connection state in a
+             * potential NAT router by sending dummy packets.
+             * RTP/RTCP dummy packets are used for RDT, too.
+             */
+            if (rtsp_st->rtp_handle &&
+                !(rt->server_type == RTSP_SERVER_WMS && i > 1))
+                ff_rtp_send_punch_packets(rtsp_st->rtp_handle);
+        }
+    }
     if (!(rt->server_type == RTSP_SERVER_REAL && rt->need_subscription)) {
         if (rt->transport == RTSP_TRANSPORT_RTP) {
             for (i = 0; i < rt->nb_rtsp_streams; i++) {