From da569556217f185830ba7412355f681a6b62f1f0 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Fri, 5 Oct 2012 01:20:55 +0200 Subject: [PATCH 1/9] ffplay: use framedrop by default when sync is not done to video When using external sync, framedrop is perfectly fine. Signed-off-by: Marton Balint --- ffplay.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffplay.c b/ffplay.c index 4223102862..b3b09b3905 100644 --- a/ffplay.c +++ b/ffplay.c @@ -1217,7 +1217,7 @@ retry: if (is->pictq_size > 1) { VideoPicture *nextvp = &is->pictq[(is->pictq_rindex + 1) % VIDEO_PICTURE_QUEUE_SIZE]; duration = nextvp->pts - vp->pts; - if((framedrop>0 || (framedrop && is->audio_st)) && time > is->frame_timer + duration){ + if((framedrop>0 || (framedrop && is->av_sync_type != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration){ is->frame_drops_late++; pictq_next_picture(is); goto retry; @@ -1535,7 +1535,7 @@ static int get_video_frame(VideoState *is, AVFrame *frame, int64_t *pts, AVPacke } if (((is->av_sync_type == AV_SYNC_AUDIO_MASTER && is->audio_st) || is->av_sync_type == AV_SYNC_EXTERNAL_CLOCK) && - (framedrop>0 || (framedrop && is->audio_st))) { + (framedrop>0 || (framedrop && is->av_sync_type != AV_SYNC_VIDEO_MASTER))) { SDL_LockMutex(is->pictq_mutex); if (is->frame_last_pts != AV_NOPTS_VALUE && *pts) { double clockdiff = get_video_clock(is) - get_master_clock(is); From eaa91ed863a72c71553c913f17c3bf2e70a11271 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Sat, 6 Oct 2012 13:31:07 +0200 Subject: [PATCH 2/9] ffplay: fix nosync threshold check in synchronize_audio Signed-off-by: Marton Balint --- ffplay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffplay.c b/ffplay.c index b3b09b3905..5c5126b48e 100644 --- a/ffplay.c +++ b/ffplay.c @@ -1899,7 +1899,7 @@ static int synchronize_audio(VideoState *is, int nb_samples) diff = get_audio_clock(is) - get_master_clock(is); - if (diff < AV_NOSYNC_THRESHOLD) { + if (fabs(diff) < AV_NOSYNC_THRESHOLD) { is->audio_diff_cum = diff + is->audio_diff_avg_coef * is->audio_diff_cum; if (is->audio_diff_avg_count < AUDIO_DIFF_AVG_NB) { /* not enough measures to have a correct estimate */ From 66bb5b1bc9b1e8e856ca20b402f968d621e7eed2 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Sat, 6 Oct 2012 19:42:42 +0200 Subject: [PATCH 3/9] ffplay: initialize audio and video pts drift This makes sensible audio and video clock values even before displaying the first frame. Signed-off-by: Marton Balint --- ffplay.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ffplay.c b/ffplay.c index 5c5126b48e..917883e34d 100644 --- a/ffplay.c +++ b/ffplay.c @@ -2677,6 +2677,8 @@ static VideoState *stream_open(const char *filename, AVInputFormat *iformat) is->continue_read_thread = SDL_CreateCond(); + is->audio_current_pts_drift = -av_gettime() / 1000000.0; + is->video_current_pts_drift = is->audio_current_pts_drift; is->av_sync_type = av_sync_type; is->read_tid = SDL_CreateThread(read_thread, is); if (!is->read_tid) { From 77bd595ad2ff339f46d63b15a9040e4b1a70b378 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Sat, 6 Oct 2012 23:55:29 +0200 Subject: [PATCH 4/9] ffplay: fix external time sync mode We now initalize the external clock to 0 and, we use the system clock to regulate the timings of audio and video in external clock sync mode. We recover from external clock sync loss, when the delay to external clock is bigger than AV_NOSYNC_THRESHOLD. Signed-off-by: Marton Balint --- ffplay.c | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/ffplay.c b/ffplay.c index 917883e34d..d9f751ca5a 100644 --- a/ffplay.c +++ b/ffplay.c @@ -153,8 +153,9 @@ typedef struct VideoState { int audio_stream; int av_sync_type; - double external_clock; /* external clock base */ - int64_t external_clock_time; + double external_clock; ///< external clock base + double external_clock_drift; ///< external clock base - time (av_gettime) at which we updated external_clock + int64_t external_clock_time; ///< last reference time double audio_clock; double audio_diff_cum; /* used for AV difference average computation */ @@ -1044,9 +1045,11 @@ static double get_video_clock(VideoState *is) /* get the current external clock value */ static double get_external_clock(VideoState *is) { - int64_t ti; - ti = av_gettime(); - return is->external_clock + ((ti - is->external_clock_time) * 1e-6); + if (is->paused) { + return is->external_clock; + } else { + return is->external_clock_drift + av_gettime() / 1000000.0; + } } /* get the current master clock value */ @@ -1070,6 +1073,19 @@ static double get_master_clock(VideoState *is) return val; } +static void update_external_clock_pts(VideoState *is, double pts) +{ + is->external_clock_time = av_gettime(); + is->external_clock = pts; + is->external_clock_drift = pts - is->external_clock_time / 1000000.0; +} + +static void check_external_clock_sync(VideoState *is, double pts) { + if (fabs(get_external_clock(is) - pts) > AV_NOSYNC_THRESHOLD) { + update_external_clock_pts(is, pts); + } +} + /* seek in the stream */ static void stream_seek(VideoState *is, int64_t pos, int64_t rel, int seek_by_bytes) { @@ -1093,6 +1109,7 @@ static void stream_toggle_pause(VideoState *is) } is->video_current_pts_drift = is->video_current_pts - av_gettime() / 1000000.0; } + update_external_clock_pts(is, get_external_clock(is)); is->paused = !is->paused; } @@ -1159,6 +1176,7 @@ static void update_video_pts(VideoState *is, double pts, int64_t pos) { is->video_current_pts_drift = is->video_current_pts - time; is->video_current_pos = pos; is->frame_last_pts = pts; + check_external_clock_sync(is, is->video_current_pts); } /* called to display each frame */ @@ -2116,6 +2134,7 @@ static void sdl_audio_callback(void *opaque, Uint8 *stream, int len) /* Let's assume the audio driver that is used by SDL has two periods. */ is->audio_current_pts = is->audio_clock - (double)(2 * is->audio_hw_buf_size + is->audio_write_buf_size) / bytes_per_sec; is->audio_current_pts_drift = is->audio_current_pts - audio_callback_time / 1000000.0; + check_external_clock_sync(is, is->audio_current_pts); } static int audio_open(void *opaque, int64_t wanted_channel_layout, int wanted_nb_channels, int wanted_sample_rate, struct AudioParams *audio_hw_params) @@ -2548,6 +2567,7 @@ static int read_thread(void *arg) packet_queue_put(&is->videoq, &flush_pkt); } } + update_external_clock_pts(is, (seek_target + ic->start_time) / (double)AV_TIME_BASE); is->seek_req = 0; eof = 0; } @@ -2677,6 +2697,7 @@ static VideoState *stream_open(const char *filename, AVInputFormat *iformat) is->continue_read_thread = SDL_CreateCond(); + update_external_clock_pts(is, 0.0); is->audio_current_pts_drift = -av_gettime() / 1000000.0; is->video_current_pts_drift = is->audio_current_pts_drift; is->av_sync_type = av_sync_type; From 2a4c7e6540c4549d6a47cd7cb288d4d910a99736 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Sun, 14 Oct 2012 17:56:32 +0200 Subject: [PATCH 5/9] ffplay: add serial field to PacketQueue entry and populate it The purpose of the serial field is to accompany the decoded data during the decoding process to know if the decoded data belongs to the data stream after the latest packet queue flush. Signed-off-by: Marton Balint --- ffplay.c | 50 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/ffplay.c b/ffplay.c index d9f751ca5a..ca4f3c2c93 100644 --- a/ffplay.c +++ b/ffplay.c @@ -87,11 +87,18 @@ const int program_birth_year = 2003; static int sws_flags = SWS_BICUBIC; +typedef struct MyAVPacketList { + AVPacket pkt; + struct MyAVPacketList *next; + int serial; +} MyAVPacketList; + typedef struct PacketQueue { - AVPacketList *first_pkt, *last_pkt; + MyAVPacketList *first_pkt, *last_pkt; int nb_packets; int size; int abort_request; + int serial; SDL_mutex *mutex; SDL_cond *cond; } PacketQueue; @@ -108,6 +115,7 @@ typedef struct VideoPicture { AVRational sample_aspect_ratio; int allocated; int reallocate; + int serial; #if CONFIG_AVFILTER AVFilterBufferRef *picref; @@ -174,6 +182,7 @@ typedef struct VideoState { int audio_write_buf_size; AVPacket audio_pkt_temp; AVPacket audio_pkt; + int audio_pkt_temp_serial; struct AudioParams audio_src; struct AudioParams audio_tgt; struct SwrContext *swr_ctx; @@ -305,16 +314,19 @@ static int packet_queue_put(PacketQueue *q, AVPacket *pkt); static int packet_queue_put_private(PacketQueue *q, AVPacket *pkt) { - AVPacketList *pkt1; + MyAVPacketList *pkt1; if (q->abort_request) return -1; - pkt1 = av_malloc(sizeof(AVPacketList)); + pkt1 = av_malloc(sizeof(MyAVPacketList)); if (!pkt1) return -1; pkt1->pkt = *pkt; pkt1->next = NULL; + if (pkt == &flush_pkt) + q->serial++; + pkt1->serial = q->serial; if (!q->last_pkt) q->first_pkt = pkt1; @@ -357,7 +369,7 @@ static void packet_queue_init(PacketQueue *q) static void packet_queue_flush(PacketQueue *q) { - AVPacketList *pkt, *pkt1; + MyAVPacketList *pkt, *pkt1; SDL_LockMutex(q->mutex); for (pkt = q->first_pkt; pkt != NULL; pkt = pkt1) { @@ -399,9 +411,9 @@ static void packet_queue_start(PacketQueue *q) } /* return < 0 if aborted, 0 if no packet and > 0 if packet. */ -static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block) +static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block, int *serial) { - AVPacketList *pkt1; + MyAVPacketList *pkt1; int ret; SDL_LockMutex(q->mutex); @@ -420,6 +432,8 @@ static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block) q->nb_packets--; q->size -= pkt1->pkt.size + sizeof(*pkt1); *pkt = pkt1->pkt; + if (serial) + *serial = pkt1->serial; av_free(pkt1); ret = 1; break; @@ -1169,7 +1183,7 @@ static void pictq_prev_picture(VideoState *is) { } } -static void update_video_pts(VideoState *is, double pts, int64_t pos) { +static void update_video_pts(VideoState *is, double pts, int64_t pos, int serial) { double time = av_gettime() / 1000000.0; /* update current video pts */ is->video_current_pts = pts; @@ -1195,7 +1209,7 @@ retry: if (is->pictq_size == 0) { SDL_LockMutex(is->pictq_mutex); if (is->frame_last_dropped_pts != AV_NOPTS_VALUE && is->frame_last_dropped_pts > is->frame_last_pts) { - update_video_pts(is, is->frame_last_dropped_pts, is->frame_last_dropped_pos); + update_video_pts(is, is->frame_last_dropped_pts, is->frame_last_dropped_pos, 0); is->frame_last_dropped_pts = AV_NOPTS_VALUE; } SDL_UnlockMutex(is->pictq_mutex); @@ -1229,7 +1243,7 @@ retry: is->frame_timer += delay * FFMAX(1, floor((time-is->frame_timer) / delay)); SDL_LockMutex(is->pictq_mutex); - update_video_pts(is, vp->pts, vp->pos); + update_video_pts(is, vp->pts, vp->pos, vp->serial); SDL_UnlockMutex(is->pictq_mutex); if (is->pictq_size > 1) { @@ -1374,7 +1388,7 @@ static void alloc_picture(VideoState *is) SDL_UnlockMutex(is->pictq_mutex); } -static int queue_picture(VideoState *is, AVFrame *src_frame, double pts1, int64_t pos) +static int queue_picture(VideoState *is, AVFrame *src_frame, double pts1, int64_t pos, int serial) { VideoPicture *vp; double frame_delay, pts = pts1; @@ -1495,6 +1509,7 @@ static int queue_picture(VideoState *is, AVFrame *src_frame, double pts1, int64_ vp->pts = pts; vp->pos = pos; vp->skip = 0; + vp->serial = serial; /* now we can update the picture count */ if (++is->pictq_windex == VIDEO_PICTURE_QUEUE_SIZE) @@ -1506,11 +1521,11 @@ static int queue_picture(VideoState *is, AVFrame *src_frame, double pts1, int64_ return 0; } -static int get_video_frame(VideoState *is, AVFrame *frame, int64_t *pts, AVPacket *pkt) +static int get_video_frame(VideoState *is, AVFrame *frame, int64_t *pts, AVPacket *pkt, int *serial) { int got_picture, i; - if (packet_queue_get(&is->videoq, pkt, 1) < 0) + if (packet_queue_get(&is->videoq, pkt, 1, serial) < 0) return -1; if (pkt->data == flush_pkt.data) { @@ -1682,6 +1697,7 @@ static int video_thread(void *arg) int64_t pts_int = AV_NOPTS_VALUE, pos = -1; double pts; int ret; + int serial = 0; #if CONFIG_AVFILTER AVCodecContext *codec = is->video_st->codec; @@ -1710,7 +1726,7 @@ static int video_thread(void *arg) avcodec_get_frame_defaults(frame); av_free_packet(&pkt); - ret = get_video_frame(is, frame, &pts_int, &pkt); + ret = get_video_frame(is, frame, &pts_int, &pkt, &serial); if (ret < 0) goto the_end; @@ -1791,11 +1807,11 @@ static int video_thread(void *arg) is->video_st->time_base.num, is->video_st->time_base.den, pts_int); } pts = pts_int * av_q2d(is->video_st->time_base); - ret = queue_picture(is, frame, pts, pos); + ret = queue_picture(is, frame, pts, pos, serial); } #else pts = pts_int * av_q2d(is->video_st->time_base); - ret = queue_picture(is, frame, pts, pkt.pos); + ret = queue_picture(is, frame, pts, pkt.pos, serial); #endif if (ret < 0) @@ -1828,7 +1844,7 @@ static int subtitle_thread(void *arg) while (is->paused && !is->subtitleq.abort_request) { SDL_Delay(10); } - if (packet_queue_get(&is->subtitleq, pkt, 1) < 0) + if (packet_queue_get(&is->subtitleq, pkt, 1, NULL) < 0) break; if (pkt->data == flush_pkt.data) { @@ -2079,7 +2095,7 @@ static int audio_decode_frame(VideoState *is, double *pts_ptr) SDL_CondSignal(is->continue_read_thread); /* read next packet */ - if ((new_packet = packet_queue_get(&is->audioq, pkt, 1)) < 0) + if ((new_packet = packet_queue_get(&is->audioq, pkt, 1, &is->audio_pkt_temp_serial)) < 0) return -1; if (pkt->data == flush_pkt.data) { From b2a8850969b89151677253be4d99e0ba29212749 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Sun, 14 Oct 2012 18:21:40 +0200 Subject: [PATCH 6/9] ffplay: only check external clock if current frame serial matches the displayed frame serial This way we avoid updating the external clocks with timestamps beloging to frames before seek. Signed-off-by: Marton Balint --- ffplay.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ffplay.c b/ffplay.c index ca4f3c2c93..c65c33aa71 100644 --- a/ffplay.c +++ b/ffplay.c @@ -1190,7 +1190,8 @@ static void update_video_pts(VideoState *is, double pts, int64_t pos, int serial is->video_current_pts_drift = is->video_current_pts - time; is->video_current_pos = pos; is->frame_last_pts = pts; - check_external_clock_sync(is, is->video_current_pts); + if (is->videoq.serial == serial) + check_external_clock_sync(is, is->video_current_pts); } /* called to display each frame */ @@ -2150,7 +2151,8 @@ static void sdl_audio_callback(void *opaque, Uint8 *stream, int len) /* Let's assume the audio driver that is used by SDL has two periods. */ is->audio_current_pts = is->audio_clock - (double)(2 * is->audio_hw_buf_size + is->audio_write_buf_size) / bytes_per_sec; is->audio_current_pts_drift = is->audio_current_pts - audio_callback_time / 1000000.0; - check_external_clock_sync(is, is->audio_current_pts); + if (is->audioq.serial == is->audio_pkt_temp_serial) + check_external_clock_sync(is, is->audio_current_pts); } static int audio_open(void *opaque, int64_t wanted_channel_layout, int wanted_nb_channels, int wanted_sample_rate, struct AudioParams *audio_hw_params) From fca16a15712b789179dfc2131506fa4762795775 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Sat, 13 Oct 2012 21:33:32 +0200 Subject: [PATCH 7/9] ffplay: add get_master_sync_type function The real av_sync_type may be different to VideoState->av_sync_type, because the required audio or video stream for audio or video clock may not be available. We will use a function to query the real av_sync_type which is used for determining the master clock. Signed-off-by: Marton Balint --- ffplay.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/ffplay.c b/ffplay.c index c65c33aa71..717051ed36 100644 --- a/ffplay.c +++ b/ffplay.c @@ -1066,23 +1066,37 @@ static double get_external_clock(VideoState *is) } } +static int get_master_sync_type(VideoState *is) { + if (is->av_sync_type == AV_SYNC_VIDEO_MASTER) { + if (is->video_st) + return AV_SYNC_VIDEO_MASTER; + else + return AV_SYNC_AUDIO_MASTER; + } else if (is->av_sync_type == AV_SYNC_AUDIO_MASTER) { + if (is->audio_st) + return AV_SYNC_AUDIO_MASTER; + else + return AV_SYNC_VIDEO_MASTER; + } else { + return AV_SYNC_EXTERNAL_CLOCK; + } +} + /* get the current master clock value */ static double get_master_clock(VideoState *is) { double val; - if (is->av_sync_type == AV_SYNC_VIDEO_MASTER) { - if (is->video_st) + switch (get_master_sync_type(is)) { + case AV_SYNC_VIDEO_MASTER: val = get_video_clock(is); - else + break; + case AV_SYNC_AUDIO_MASTER: val = get_audio_clock(is); - } else if (is->av_sync_type == AV_SYNC_AUDIO_MASTER) { - if (is->audio_st) - val = get_audio_clock(is); - else - val = get_video_clock(is); - } else { - val = get_external_clock(is); + break; + default: + val = get_external_clock(is); + break; } return val; } From d30c69251f04bf56d2a6f0d3dabd5ea1561dc3f7 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Sat, 13 Oct 2012 22:31:17 +0200 Subject: [PATCH 8/9] ffplay: use get_master_sync_type where necessary We should make decisions based on the real sync type. This also simplifies the code. Signed-off-by: Marton Balint --- ffplay.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ffplay.c b/ffplay.c index 717051ed36..95a6ac4635 100644 --- a/ffplay.c +++ b/ffplay.c @@ -1146,8 +1146,7 @@ static double compute_target_delay(double delay, VideoState *is) double sync_threshold, diff; /* update delay to follow master synchronisation source */ - if (((is->av_sync_type == AV_SYNC_AUDIO_MASTER && is->audio_st) || - is->av_sync_type == AV_SYNC_EXTERNAL_CLOCK)) { + if (get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER) { /* if video is slave, we try to correct big delays by duplicating or deleting a frame */ diff = get_video_clock(is) - get_master_clock(is); @@ -1264,7 +1263,7 @@ retry: if (is->pictq_size > 1) { VideoPicture *nextvp = &is->pictq[(is->pictq_rindex + 1) % VIDEO_PICTURE_QUEUE_SIZE]; duration = nextvp->pts - vp->pts; - if((framedrop>0 || (framedrop && is->av_sync_type != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration){ + if((framedrop>0 || (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration){ is->frame_drops_late++; pictq_next_picture(is); goto retry; @@ -1582,8 +1581,7 @@ static int get_video_frame(VideoState *is, AVFrame *frame, int64_t *pts, AVPacke *pts = 0; } - if (((is->av_sync_type == AV_SYNC_AUDIO_MASTER && is->audio_st) || is->av_sync_type == AV_SYNC_EXTERNAL_CLOCK) && - (framedrop>0 || (framedrop && is->av_sync_type != AV_SYNC_VIDEO_MASTER))) { + if (framedrop>0 || (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) { SDL_LockMutex(is->pictq_mutex); if (is->frame_last_pts != AV_NOPTS_VALUE && *pts) { double clockdiff = get_video_clock(is) - get_master_clock(is); @@ -1941,8 +1939,7 @@ static int synchronize_audio(VideoState *is, int nb_samples) int wanted_nb_samples = nb_samples; /* if not master, then we try to remove or add samples to correct the clock */ - if (((is->av_sync_type == AV_SYNC_VIDEO_MASTER && is->video_st) || - is->av_sync_type == AV_SYNC_EXTERNAL_CLOCK)) { + if (get_master_sync_type(is) != AV_SYNC_AUDIO_MASTER) { double diff, avg_diff; int min_nb_samples, max_nb_samples; From 3166a6fc379789b3782f431dd232033c2069c443 Mon Sep 17 00:00:00 2001 From: Marton Balint Date: Sun, 14 Oct 2012 00:47:15 +0200 Subject: [PATCH 9/9] ffplay: if there is no audio stream, use external clock by default Otherwise playing the video could be much slower than realtime if the system can't decode or display the frames fast enough. Signed-off-by: Marton Balint --- ffplay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffplay.c b/ffplay.c index 95a6ac4635..c6cf880ccc 100644 --- a/ffplay.c +++ b/ffplay.c @@ -1076,7 +1076,7 @@ static int get_master_sync_type(VideoState *is) { if (is->audio_st) return AV_SYNC_AUDIO_MASTER; else - return AV_SYNC_VIDEO_MASTER; + return AV_SYNC_EXTERNAL_CLOCK; } else { return AV_SYNC_EXTERNAL_CLOCK; }