diff --git a/libavfilter/af_atempo.c b/libavfilter/af_atempo.c
index 4621b67b03..654b080e89 100644
--- a/libavfilter/af_atempo.c
+++ b/libavfilter/af_atempo.c
@@ -531,21 +531,21 @@ static int yae_load_frag(ATempoContext *atempo,
     dst = frag->data;
 
     start = atempo->position[0] - atempo->size;
-    zeros = 0;
 
-    if (frag->position[0] < start) {
-        // what we don't have we substitute with zeros:
-        zeros = FFMIN(start - frag->position[0], (int64_t)nsamples);
-        av_assert0(zeros != nsamples);
-
-        memset(dst, 0, zeros * atempo->stride);
-        dst += zeros * atempo->stride;
-    }
+    // what we don't have we substitute with zeros:
+    zeros =
+      frag->position[0] < start ?
+      FFMIN(start - frag->position[0], (int64_t)nsamples) : 0;
 
     if (zeros == nsamples) {
         return 0;
     }
 
+    if (frag->position[0] < start) {
+        memset(dst, 0, zeros * atempo->stride);
+        dst += zeros * atempo->stride;
+    }
+
     // get the remaining data from the ring buffer:
     na = (atempo->head < atempo->tail ?
           atempo->tail - atempo->head :