Implement audio video synchronization

pjmedia/include/pjmedia/av_sync.h

@@ -64,9 +64,9 @@ PJ_BEGIN_DECL
  * 2. Update presentation time whenever returns a frame (to play/render).
  * 3. Add mechanism to calculate current delay & optimal delay from
  *    jitter buffer size.
- * 4. When AV sync request to speed up, as long as the end delay is
+ * 4. When AV sync requests to speed up, as long as the end delay is
  *    not lower than the optimal delay, do it.
- * 5. When AV sync request to slow down, do it, i.e: increase and maintain
+ * 5. When AV sync requests to slow down, do it, i.e: increase and maintain
  *    the jitter buffer size.
  */
 

pjmedia/include/pjmedia/jbuf.h

@@ -102,6 +102,7 @@ typedef struct pjmedia_jb_state
     unsigned    min_prefetch;       /**< Minimum allowed prefetch, in frms. */
     unsigned    max_prefetch;       /**< Maximum allowed prefetch, in frms. */
     unsigned    max_count;          /**< Jitter buffer capacity, in frames. */
+    unsigned    min_delay_set;      /**< Minimum delay setting, in frames.  */
 
     /* Status */
     unsigned    burst;              /**< Current burst level, in frames     */
@@ -474,6 +475,20 @@ PJ_DECL(pj_status_t) pjmedia_jbuf_get_state( const pjmedia_jbuf *jb,
                                              pjmedia_jb_state *state );
 
 
+/**
+ * Set minimum delay of the jitter buffer. Normally jitter buffer tries to
+ * maintain the optimal delay calculated based on current burst level.
+ * When the minimum delay is set, the jitter buffer will adjust the delay to
+ * the greater of the optimal delay and the minimum delay.
+ *
+ * @param jb            The jitter buffer.
+ * @param min_delay     The minimum delay, in millisecond.
+ *
+ * @return              PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t) pjmedia_jbuf_set_min_delay(pjmedia_jbuf *jb,
+                                                unsigned min_delay);
+
 
 PJ_END_DECL
 

pjmedia/src/pjmedia/av_sync.c

@@ -27,21 +27,15 @@
  * When delay is higher than this setting, some actions will be required,
  * i.e: speed-up or slow-down media playback.
  */
-#define MAX_DELAY_MS                    150
+#define MAX_DELAY_MS                    50
 
  /* Maximum number of request to a media to speeding up its delay to match
   * to the fastest media, before requesting the fastest media to slow down.
   */
 #define MAX_DELAY_REQ_CNT               10
 
-/* Minimum delay progress after last delay adjustment request. If the progress
- * is less than this, AV sync may request the fastest media to slow down.
- */
-#define MIN_SPEED_UP_PROGRESS_MS        20
-
 /* Enable/disable trace */
-#define TRACE_ENABLED                   1
-#if TRACE_ENABLED
+#if 1
 #  define TRACE_(x) PJ_LOG(1, x)
 #else
 #  define TRACE_(x)
@@ -64,6 +58,7 @@ struct pjmedia_av_sync_media
 
     /* Last presentation timestamp */
     pj_timestamp             last_ntp;          /* Last PTS, in NTP units   */
+    pj_int32_t               smooth_diff;
 
     /* Delay adjustment requested to this media */
     pj_int32_t               last_adj_delay_req; /* Last requested delay    */
@@ -240,6 +235,7 @@ PJ_DEF(pj_status_t) pjmedia_av_sync_del_media(
     media->is_ref_set = PJ_FALSE;
     media->last_adj_delay_req = 0;
     media->adj_delay_req_cnt = 0;
+    media->smooth_diff = 0;
 
     pj_list_push_back(&avs->free_media_list, media);
     pj_grp_lock_release(avs->grp_lock);
@@ -317,7 +313,7 @@ PJ_DEF(pj_int32_t) pjmedia_av_sync_update_pts(
             media->last_adj_delay_req = avs->slowdown_req_ms;
             media->adj_delay_req_cnt = 0;
             avs->slowdown_req_ms = 0;
-            TRACE_((avs->pool->obj_name, "%s requested to slow down %dms",
+            TRACE_((avs->pool->obj_name, "%s is requested to slow down %dms",
                     media->name, media->last_adj_delay_req));
             return media->last_adj_delay_req;
         }
@@ -330,11 +326,15 @@ PJ_DEF(pj_int32_t) pjmedia_av_sync_update_pts(
         pj_sub_timestamp(&ntp_diff, &media->last_ntp);
         ms_diff = ntp_to_ms(&ntp_diff);
 
+        /* Smoothen and round down the delay */
+        ms_diff = ((ms_diff + 19 * media->smooth_diff) / 200) * 10;
+        media->smooth_diff = ms_diff;
+
         /* The delay is tolerable, just return 0 */
-        if (ms_diff < MAX_DELAY_MS) {
+        if (ms_diff <= MAX_DELAY_MS) {
             if (media->last_adj_delay_req) {
                 TRACE_((avs->pool->obj_name,
-                        "%s speed up completed, delay looks good=%ums",
+                        "%s speeds up completed, delay looks good=%ums",
                         media->name, ms_diff));
             }
             /* Reset the request delay & counter */
@@ -344,40 +344,53 @@ PJ_DEF(pj_int32_t) pjmedia_av_sync_update_pts(
         }
 
         /* Check if any speed-up request has been done before */
-        if (media->last_adj_delay_req < 0) {
-            pj_int32_t progress;
-
-            /* Check if the previous delay request has shown some progress */
-            progress = (-media->last_adj_delay_req) - ms_diff;
-            if (progress >= MIN_SPEED_UP_PROGRESS_MS) {
-                /* Yes, let's just request again and wait */
-                TRACE_((avs->pool->obj_name,
-                        "%s speed up in progress, current delay=%ums",
-                        media->name, ms_diff));
-            }
-
+        if (media->last_adj_delay_req) {
             /* Check if request number has reached limit */
-            else if (media->adj_delay_req_cnt >= MAX_DELAY_REQ_CNT) {
+            if (media->adj_delay_req_cnt >= MAX_DELAY_REQ_CNT) {
                 /* After several requests this media still cannot catch up,
                  * signal the synchronizer to slow down the fastest media.
                  */
                 if (avs->slowdown_req_ms < ms_diff)
                     avs->slowdown_req_ms = ms_diff;
 
-                /* Reset the request counter */
+                TRACE_((avs->pool->obj_name,
+                        "%s request limit has been reached, requesting "
+                        "the fastest media to slow down by %ums",
+                        media->name, avs->slowdown_req_ms));
+
+                /* Reset the request counter.
+                 * And still keep requesting for speed up, shouldn't we?
+                 */
                 media->adj_delay_req_cnt = 0;
+            } else {
+                pj_int32_t progress, min_expected;
+
+                /* Check if the previous delay request has shown some
+                 * progress.
+                 */
+                progress = (-media->last_adj_delay_req) - ms_diff;
+                min_expected = -media->last_adj_delay_req /
+                           (MAX_DELAY_REQ_CNT - media->adj_delay_req_cnt + 1);
+                if (progress >= min_expected) {
+                    /* Yes, let's just request again and wait */
+                    TRACE_((avs->pool->obj_name,
+                            "%s speeds up in progress, current delay=%ums",
+                            media->name, ms_diff));
+                }
             }
         } else {
             /* First request to speed up */
             media->adj_delay_req_cnt = 0;
         }
 
         /* Request the media to speed up & increment the counter */
-        TRACE_((avs->pool->obj_name,
-                "%s requested to speed up #%d %ums",
-                media->name, media->adj_delay_req_cnt, ms_diff));
         media->last_adj_delay_req = -(pj_int32_t)ms_diff;
         media->adj_delay_req_cnt++;
+
+        TRACE_((avs->pool->obj_name,
+                "%s is requested to speed up #%d %ums",
+                media->name, media->adj_delay_req_cnt, ms_diff));
+
         return media->last_adj_delay_req;
     }
 

pjmedia/src/pjmedia/jbuf.c

@@ -110,6 +110,7 @@ struct pjmedia_jbuf
                                              calculation                    */
     int             jb_min_shrink_gap;  /**< How often can we shrink        */
     discard_algo    jb_discard_algo;    /**< Discard algorithm              */
+    unsigned        jb_min_delay;       /**< Minimum delay, in frames       */
 
     /* Buffer */
     jb_framelist_t  jb_framelist;       /**< the buffer                     */
@@ -857,9 +858,12 @@ static void jbuf_discard_static(pjmedia_jbuf *jb)
      * so just disable it when progressive discard is active.
      */
     int diff, burst_level;
+    unsigned ref_size;
 
     burst_level = PJ_MAX(jb->jb_eff_level, jb->jb_level);
-    diff = jb_framelist_eff_size(&jb->jb_framelist) - burst_level*2;
+    ref_size = PJ_MAX(jb_framelist_eff_size(&jb->jb_framelist),
+                      jb->jb_min_delay);
+    diff = ref_size - burst_level*2;
 
     if (diff >= STA_DISC_SAFE_SHRINKING_DIFF) {
         int seq_origin;
@@ -891,17 +895,18 @@ static void jbuf_discard_static(pjmedia_jbuf *jb)
 
 static void jbuf_discard_progressive(pjmedia_jbuf *jb)
 {
-    unsigned cur_size, burst_level, overflow, T, discard_dist;
+    unsigned ref_size, burst_level, overflow, T, discard_dist;
     int last_seq;
 
     /* Should be done in PUT operation */
     if (jb->jb_last_op != JB_OP_PUT)
         return;
 
     /* Check if latency is longer than burst */
-    cur_size = jb_framelist_eff_size(&jb->jb_framelist);
+    ref_size = PJ_MAX(jb_framelist_eff_size(&jb->jb_framelist),
+                      jb->jb_min_delay);
     burst_level = PJ_MAX(jb->jb_eff_level, jb->jb_level);
-    if (cur_size <= burst_level) {
+    if (ref_size <= burst_level) {
         /* Reset any scheduled discard */
         jb->jb_discard_dist = 0;
         return;
@@ -919,7 +924,7 @@ static void jbuf_discard_progressive(pjmedia_jbuf *jb)
             (PJMEDIA_JBUF_PRO_DISC_MAX_BURST-PJMEDIA_JBUF_PRO_DISC_MIN_BURST);
 
     /* Calculate current discard distance */
-    overflow = cur_size - burst_level;
+    overflow = ref_size - burst_level;
     discard_dist = T * jb->jb_frame_ptime_denum / overflow /
                    jb->jb_frame_ptime;
 
@@ -1146,6 +1151,10 @@ PJ_DEF(void) pjmedia_jbuf_get_frame3(pjmedia_jbuf *jb,
 
         jb->jb_empty++;
 
+    } else if (jb_framelist_eff_size(&jb->jb_framelist) < jb->jb_min_delay) {
+        *p_frame_type = PJMEDIA_JB_MISSING_FRAME;
+        if (size)
+            *size = 0;
     } else {
 
         pjmedia_jb_frame_type ftype = PJMEDIA_JB_NORMAL_FRAME;
@@ -1205,6 +1214,7 @@ PJ_DEF(pj_status_t) pjmedia_jbuf_get_state( const pjmedia_jbuf *jb,
     state->min_prefetch = jb->jb_min_prefetch;
     state->max_prefetch = jb->jb_max_prefetch;
     state->max_count = (unsigned)jb->jb_max_count;
+    state->min_delay_set = jb->jb_min_delay;
 
     state->burst = jb->jb_eff_level;
     state->prefetch = jb->jb_prefetch;
@@ -1270,3 +1280,22 @@ PJ_DEF(unsigned) pjmedia_jbuf_remove_frame(pjmedia_jbuf *jb,
 
     return count;
 }
+
+
+PJ_DEF(pj_status_t) pjmedia_jbuf_set_min_delay(pjmedia_jbuf *jb,
+                                               unsigned min_delay)
+{
+    PJ_ASSERT_RETURN(jb, PJ_EINVAL);
+
+    /* Convert milliseconds to frames */
+    min_delay *= jb->jb_frame_ptime_denum;
+    jb->jb_min_delay = min_delay / jb->jb_frame_ptime;
+    if (min_delay % jb->jb_frame_ptime)
+        jb->jb_min_delay++;
+
+    /* Should not be higher than half of jitter buffer capacity */
+    if (jb->jb_min_delay > jb->jb_max_count/2)
+        jb->jb_min_delay = jb->jb_max_count/2;
+
+    return PJ_SUCCESS;
+}

pjmedia/src/pjmedia/rtcp.c

@@ -948,7 +948,7 @@ PJ_DEF(void) pjmedia_rtcp_build_rtcp(pjmedia_rtcp_session *sess,
      * sent RTCP SR.
      */
     if (sess->stat.tx.pkt != pj_ntohl(sess->rtcp_sr_pkt.sr.sender_pcount)) {
-        pj_time_val ts_time;
+        //pj_time_val ts_time;
         pj_uint32_t rtp_ts;
 
         /* So we should send RTCP SR */
@@ -968,11 +968,13 @@ PJ_DEF(void) pjmedia_rtcp_build_rtcp(pjmedia_rtcp_session *sess,
         sr->ntp_frac = pj_htonl(ntp.lo);
 
         /* Fill in RTP timestamp (corresponds to NTP timestamp) in SR. */
-        ts_time.sec = ntp.hi - sess->tv_base.sec - JAN_1970;
-        ts_time.msec = (long)(ntp.lo * 1000.0 / 0xFFFFFFFF);
-        rtp_ts = sess->rtp_ts_base +
-                 (pj_uint32_t)(sess->clock_rate*ts_time.sec) +
-                 (pj_uint32_t)(sess->clock_rate*ts_time.msec/1000);
+        // Use real last transmitted RTP timestamp instead of calculated one.
+        //ts_time.sec = ntp.hi - sess->tv_base.sec - JAN_1970;
+        //ts_time.msec = (long)(ntp.lo * 1000.0 / 0xFFFFFFFF);
+        //rtp_ts = sess->rtp_ts_base +
+        //         (pj_uint32_t)(sess->clock_rate*ts_time.sec) +
+        //         (pj_uint32_t)(sess->clock_rate*ts_time.msec/1000);
+        rtp_ts = sess->stat.rtp_tx_last_ts;
         sr->rtp_ts = pj_htonl(rtp_ts);
 
         TRACE_((sess->name, "TX RTCP SR: ntp_ts=%p", 

pjmedia/src/pjmedia/stream.c

@@ -491,14 +491,36 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame)
             if (!use_dec_buf)
                 samples_count += samples_per_frame;
 
-            /* Update synchronizer with presentation time */
+            /* Update synchronizer with presentation time and check if the
+             * synchronizer requests for delay adjustment.
+             */
             if (c_strm->av_sync_media) {
                 pj_timestamp pts = { 0 };
-                pj_int32_t delay_req;
+                pj_int32_t delay_req_ms;
 
                 pts.u32.lo = rtp_ts;
-                delay_req = pjmedia_av_sync_update_pts(c_strm->av_sync_media,
+                delay_req_ms = pjmedia_av_sync_update_pts(c_strm->av_sync_media,
                                                        &pts);
+                if (delay_req_ms) {
+                    /* Delay adjustment is requested */
+                    pjmedia_jb_state jb_state;
+                    int target_delay_ms, cur_delay_ms;
+
+                    /* Increase delay slowly, but decrease delay quickly */
+                    if (delay_req_ms > 0)
+                        delay_req_ms = delay_req_ms * 3 / 4;
+                    else 
+                        delay_req_ms = delay_req_ms * 4 / 3;
+
+                    /* Apply delay request to jitter buffer */
+                    pjmedia_jbuf_get_state(c_strm->jb, &jb_state);
+                    cur_delay_ms = jb_state.min_delay_set * stream->dec_ptime/
+                                   stream->dec_ptime_denum;
+                    target_delay_ms = cur_delay_ms + delay_req_ms;
+                    if (target_delay_ms < 0)
+                        target_delay_ms = 0;
+                    pjmedia_jbuf_set_min_delay(c_strm->jb, target_delay_ms);
+                }
             }
         }
     }

pjsip/src/pjsua-lib/pjsua_aud.c

@@ -784,7 +784,8 @@ pj_status_t pjsua_aud_channel_update(pjsua_call_media *call_med,
         /* Add stream to synchronizer */
         if (call->av_sync) {
             status = pjmedia_stream_common_set_avsync(
-                                    call_med->strm.a.stream, call->av_sync);
+                            (pjmedia_stream_common*)call_med->strm.a.stream,
+                            call->av_sync);
             if (status != PJ_SUCCESS)
                 goto on_return;
         }

pjsip/src/pjsua-lib/pjsua_vid.c

@@ -1188,7 +1188,8 @@ pj_status_t pjsua_vid_channel_update(pjsua_call_media *call_med,
         /* Add stream to synchronizer */
         if (call->av_sync) {
             status = pjmedia_stream_common_set_avsync(
-                                    call_med->strm.v.stream, call->av_sync);
+                            (pjmedia_stream_common*)call_med->strm.v.stream,
+                            call->av_sync);
             if (status != PJ_SUCCESS)
                 goto on_error;
         }