diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index 648941e57f..fc3e4f0f88 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -2284,7 +2284,7 @@ static int matroska_parse_block(MatroskaDemuxContext *matroska, uint8_t *data,
         return res;
     av_assert1(block_duration != AV_NOPTS_VALUE);
 
-    block_time = AV_RB16(data);
+    block_time = sign_extend(AV_RB16(data), 16);
     data += 2;
     flags = *data++;
     size -= 3;