diff --git a/libavformat/internal.h b/libavformat/internal.h
index 9b07cfb271..23757dc4fc 100644
--- a/libavformat/internal.h
+++ b/libavformat/internal.h
@@ -310,6 +310,13 @@ typedef struct FFStream {
      */
     int64_t mux_ts_offset;
 
+    /**
+     * This is the lowest ts allowed in this track; it may be set by the muxer
+     * during init or write_header and influences the automatic timestamp
+     * shifting code.
+     */
+    int64_t lowest_ts_allowed;
+
     /**
      * Internal data to check for wrapping of the time stamp
      */
diff --git a/libavformat/mux.c b/libavformat/mux.c
index a3b50dadb6..5d89458f82 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -632,6 +632,8 @@ static void handle_avoid_negative_ts(FFFormatContext *si, FFStream *sti,
         if (ts == AV_NOPTS_VALUE)
             return;
 
+        ts -= sti->lowest_ts_allowed;
+
         /* Peek into the muxing queue to improve our estimate
          * of the lowest timestamp if av_interleaved_write_frame() is used. */
         for (const PacketListEntry *pktl = si->packet_buffer.head;
@@ -640,6 +642,7 @@ static void handle_avoid_negative_ts(FFFormatContext *si, FFStream *sti,
             int64_t cmp_ts = use_pts ? pktl->pkt.pts : pktl->pkt.dts;
             if (cmp_ts == AV_NOPTS_VALUE)
                 continue;
+            cmp_ts -= ffstream(s->streams[pktl->pkt.stream_index])->lowest_ts_allowed;
             if (s->output_ts_offset)
                 cmp_ts += av_rescale_q(s->output_ts_offset, AV_TIME_BASE_Q, cmp_tb);
             if (av_compare_ts(cmp_ts, cmp_tb, ts, tb) < 0) {
@@ -669,7 +672,7 @@ static void handle_avoid_negative_ts(FFFormatContext *si, FFStream *sti,
         pkt->pts += offset;
 
     if (si->avoid_negative_ts_use_pts) {
-        if (pkt->pts != AV_NOPTS_VALUE && pkt->pts < 0) {
+        if (pkt->pts != AV_NOPTS_VALUE && pkt->pts < sti->lowest_ts_allowed) {
             av_log(s, AV_LOG_WARNING, "failed to avoid negative "
                    "pts %s in stream %d.\n"
                    "Try -avoid_negative_ts 1 as a possible workaround.\n",
@@ -678,7 +681,7 @@ static void handle_avoid_negative_ts(FFFormatContext *si, FFStream *sti,
             );
         }
     } else {
-        if (pkt->dts != AV_NOPTS_VALUE && pkt->dts < 0) {
+        if (pkt->dts != AV_NOPTS_VALUE && pkt->dts < sti->lowest_ts_allowed) {
             av_log(s, AV_LOG_WARNING,
                    "Packets poorly interleaved, failed to avoid negative "
                    "timestamp %s in stream %d.\n"