Implement audio video synchronization

pjmedia/build/Makefile

@@ -58,7 +58,7 @@ export _LDFLAGS := $(APP_THIRD_PARTY_LIBS) \
 #
 export PJMEDIA_SRCDIR = ../src/pjmedia
 export PJMEDIA_OBJS += $(OS_OBJS) $(M_OBJS) $(CC_OBJS) $(HOST_OBJS) \
-			alaw_ulaw.o alaw_ulaw_table.o avi_player.o \
+			alaw_ulaw.o alaw_ulaw_table.o avi_player.o av_sync.o \
 			bidirectional.o clock_thread.o codec.o conference.o \
 			conf_switch.o converter.o  converter_libswscale.o converter_libyuv.o \
 			delaybuf.o echo_common.o \

pjmedia/build/pjmedia.vcxproj

@@ -610,6 +610,7 @@
     <ClCompile Include="..\src\pjmedia\alaw_ulaw_table.c" />
     <ClCompile Include="..\src\pjmedia\audiodev.c" />
     <ClCompile Include="..\src\pjmedia\avi_player.c" />
+    <ClCompile Include="..\src\pjmedia\av_sync.c" />
     <ClCompile Include="..\src\pjmedia\bidirectional.c" />
     <ClCompile Include="..\src\pjmedia\clock_thread.c" />
     <ClCompile Include="..\src\pjmedia\codec.c" />
@@ -738,6 +739,7 @@
     <ClInclude Include="..\include\pjmedia\audiodev.h" />
     <ClInclude Include="..\include\pjmedia\avi.h" />
     <ClInclude Include="..\include\pjmedia\avi_stream.h" />
+    <ClInclude Include="..\include\pjmedia\av_sync.h" />
     <ClInclude Include="..\include\pjmedia\bidirectional.h" />
     <ClInclude Include="..\include\pjmedia\circbuf.h" />
     <ClInclude Include="..\include\pjmedia\clock.h" />

pjmedia/build/pjmedia.vcxproj.filters

@@ -230,6 +230,9 @@
     <ClCompile Include="..\src\pjmedia\echo_webrtc_aec3.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\src\pjmedia\av_sync.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\include\pjmedia\alaw_ulaw.h">
@@ -418,5 +421,8 @@
     <ClInclude Include="..\include\pjmedia\vid_conf.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\include\pjmedia\av_sync.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

pjmedia/include/pjmedia/av_sync.h

@@ -0,0 +1,215 @@
+/*
+ * Copyright (C) 2025 Teluu Inc. (http://www.teluu.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+#ifndef __PJMEDIA_AV_SYNC_H__
+#define __PJMEDIA_AV_SYNC_H__
+
+ /**
+  * @file av_sync.h
+  * @brief Inter-media Synchronization.
+  */
+#include <pjmedia/types.h>
+
+
+PJ_BEGIN_DECL
+
+
+/**
+ * @defgroup PJMEDIA_AV_SYNC Inter-media Synchronization
+ * @ingroup PJMEDIA_SESSION
+ * @brief Synchronize presentation time of multiple media in a session.
+ * @{
+ *
+ * A call session may consist of multiple media, e.g: some audio and some
+ * video, which frequently have different delays when presented in the
+ * receiver side. This module synchronizes all media in the same session
+ * based on NTP timestamp & RTP timestamp info provided by the sender in
+ * RTCP SR.
+ *
+ * Here are steps to use this module:
+ * 1. Create AV sync using #pjmedia_av_sync_create().
+ * 2. Adds all media to be synchronized using #pjmedia_av_sync_add_media().
+ * 3. Call #pjmedia_av_sync_update_ref() each time the media receiving
+ *    an RTCP SR packet.
+ * 4. Call #pjmedia_av_sync_update_pts() each time the media returning
+ *    a frame to be presented, e.g: via port.get_frame(). The function may
+ *    request the media to adjust its delay.
+ * 5. Call #pjmedia_av_sync_del_media() when a media is removed from the
+ *    session.
+ * 6. Call #pjmedia_av_sync_destroy() when the session is ended.
+ * 
+ * The primary synchronization logic is implemented within the
+ * #pjmedia_av_sync_update_pts() function. This function will calculate
+ * the lag between the calling media to the earliest media and will provide
+ * a feedback to the calling media whether it is in synchronized state,
+ * late, or early so the media can respond accordingly.
+ * Initially this function will try to request slower media to speed up.
+ * If after a specific number of requests (i.e: configurable via
+ * PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT) and the lag is still beyond a tolerable
+ * value (i.e: configurable via PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC), the
+ * function will issue slow down request to the fastest media.
+ */
+
+
+/**
+ * Inter-media synchronizer, opaque.
+ */
+typedef struct pjmedia_av_sync pjmedia_av_sync;
+
+
+/**
+ * Media synchronization handle, opaque.
+ */
+typedef struct pjmedia_av_sync_media pjmedia_av_sync_media;
+
+
+/**
+ * Media settings.
+ */
+typedef struct {
+    /**
+     * Name of the media
+     */
+    char                       *name;
+
+    /**
+     * Media clock rate or sampling rate.
+     */
+    unsigned                    clock_rate;
+} pjmedia_av_sync_media_setting;
+
+
+/**
+ * Get default settings for media.
+ *
+ * @param setting           The media setting.
+ */
+PJ_DECL(void) pjmedia_av_sync_media_setting_default(
+                                pjmedia_av_sync_media_setting *setting);
+
+/**
+ * Create media synchronizer.
+ *
+ * @param endpt             The media endpoint.
+ * @param option            Synchronization option, must be NULL for now.
+ * @param av_sync           The pointer to receive the media synchronizer.
+ *
+ * @return                  PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_create(
+                                pjmedia_endpt *endpt,
+                                const void *option,
+                                pjmedia_av_sync **av_sync);
+
+
+/**
+ * Destroy media synchronizer.
+ *
+ * @param av_sync           The media synchronizer.
+ */
+PJ_DECL(void) pjmedia_av_sync_destroy(pjmedia_av_sync *av_sync);
+
+
+/**
+ * Add a media to synchronizer.
+ *
+ * @param av_sync           The media synchronizer.
+ * @param setting           The media setting.
+ * @param av_sync_media     The pointer to receive the media synchronization
+ *                          handle.
+ *
+ * @return                  PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_add_media(
+                                pjmedia_av_sync* av_sync,
+                                const pjmedia_av_sync_media_setting *setting,
+                                pjmedia_av_sync_media **av_sync_media);
+
+
+/**
+ * Remove a media from synchronizer.
+ *
+ * @param av_sync           The media synchronizer.
+ * @param av_sync_media     The media synchronization handle.
+ *
+ * @return                  PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_del_media(
+                                pjmedia_av_sync *av_sync,
+                                pjmedia_av_sync_media *av_sync_media);
+
+
+/**
+ * Update synchronizer about the last presentation timestamp of the specified
+ * media. Normally this function is called when media is producing a frame
+ * to be rendered (e.g: in port's get_frame() method). Upon returning, the
+ * media may be requested to adjust its delay so it matches to the earliest
+ * or the latest media, i.e: by speeding up or slowing down.
+ *
+ * Initially this function will try to request slower media to speed up.
+ * If after a specific number of requests (i.e: configurable via
+ * PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT) and the lag is still beyond a tolerable
+ * value (i.e: configurable via PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC), the
+ * function will issue slow down request to the fastest media.
+ *
+ * Note that currently audio stream can adjust delay using
+ * #pjmedia_jbuf_set_min_delay(), while video stream does not have the
+ * capability to adjust delay yet.
+ *
+ * @param av_sync_media     The media synchronization handle.
+ * @param pts               The presentation timestamp.
+ * @param adjust_delay      Optional pointer to receive adjustment delay
+ *                          required, in milliseconds, to make this media
+ *                          synchronized to the fastest media.
+ *                          Possible output values are:
+ *                          0 when no action is needed,
+ *                          possitive value when increasing delay is needed,
+ *                          or negative value when decreasing delay is needed.
+ *
+ * @return                  PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_update_pts(
+                                pjmedia_av_sync_media *av_sync_media,
+                                const pj_timestamp *pts,
+                                pj_int32_t *adjust_delay);
+
+
+/**
+ * Update synchronizer about reference timestamps of the specified media.
+ * Normally this function is called after a media receives RTCP SR packet.
+ *
+ * @param av_sync_media     The media synchronization handle.
+ * @param ntp               The NTP timestamp info from RTCP SR.
+ * @param ts                The RTP timestamp info from RTCP SR.
+ *
+ * @return                  PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_update_ref(
+                                pjmedia_av_sync_media *av_sync_media,
+                                const pj_timestamp *ntp,
+                                const pj_timestamp *ts);
+
+
+/**
+ * @}
+ */
+
+
+PJ_END_DECL
+
+
+#endif  /* __PJMEDIA_AV_SYNC_H__ */

pjmedia/include/pjmedia/config.h

@@ -1734,6 +1734,29 @@
 #undef PJMEDIA_VID_STREAM_CHECK_RTP_PT
 #define PJMEDIA_VID_STREAM_CHECK_RTP_PT      PJMEDIA_STREAM_CHECK_RTP_PT
 
+
+/**
+ * Maximum tolerable presentation lag from the earliest to the latest media,
+ * in milliseconds, in inter-media synchronization. When the delay is
+ * higher than this setting, the media synchronizer will request the slower
+ * media to speed up. And if after a number of speed up requests the delay
+ * is still beyond this setting, the fastest media will be requested to
+ * slow down.
+ *
+ * Default: 45 ms
+ */
+#define PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC   45
+
+
+/**
+  * Maximum number of speed up request to synchronize presentation time,
+  * before a slow down request to the fastest media is issued.
+  *
+  * Default: 10
+  */
+#define PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT      10
+
+
 /**
  * @}
  */

pjmedia/include/pjmedia/jbuf.h

@@ -102,6 +102,7 @@ typedef struct pjmedia_jb_state
     unsigned    min_prefetch;       /**< Minimum allowed prefetch, in frms. */
     unsigned    max_prefetch;       /**< Maximum allowed prefetch, in frms. */
     unsigned    max_count;          /**< Jitter buffer capacity, in frames. */
+    unsigned    min_delay_set;      /**< Minimum delay setting, in frames.  */
 
     /* Status */
     unsigned    burst;              /**< Current burst level, in frames     */
@@ -474,6 +475,20 @@ PJ_DECL(pj_status_t) pjmedia_jbuf_get_state( const pjmedia_jbuf *jb,
                                              pjmedia_jb_state *state );
 
 
+/**
+ * Set minimum delay of the jitter buffer. Normally jitter buffer tries to
+ * maintain the optimal delay calculated based on current burst level.
+ * When the minimum delay is set, the jitter buffer will adjust the delay to
+ * the greater of the optimal delay and the minimum delay.
+ *
+ * @param jb            The jitter buffer.
+ * @param min_delay     The minimum delay, in millisecond.
+ *
+ * @return              PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t) pjmedia_jbuf_set_min_delay(pjmedia_jbuf *jb,
+                                                unsigned min_delay);
+
 
 PJ_END_DECL
 

pjmedia/include/pjmedia/rtcp.h

@@ -263,6 +263,10 @@ typedef struct pjmedia_rtcp_session
 
     pj_uint32_t             rx_lsr;     /**< NTP ts in last SR received     */
     pj_timestamp            rx_lsr_time;/**< Time when last SR is received  */
+    pj_uint32_t             rx_lsr_ts;  /**< RTP ts in last SR received     */
+    pj_timestamp            rx_lsr_ntp; /**< Original/64bit NTP ts in last
+                                             SR received                    */
+
     pj_uint32_t             peer_ssrc;  /**< Peer SSRC                      */
 
     pjmedia_rtcp_stat       stat;       /**< Bidirectional stream stat.     */

pjmedia/include/pjmedia/stream_common.h

@@ -24,6 +24,7 @@
  * @brief Stream common functions.
  */
 
+#include <pjmedia/av_sync.h>
 #include <pjmedia/codec.h>
 #include <pjmedia/jbuf.h>
 #include <pjmedia/sdp.h>
@@ -151,6 +152,11 @@ typedef struct pjmedia_stream_common
     int                      pending_rtcp_fb_nack;  /**< Any pending NACK?  */
     pjmedia_rtcp_fb_nack     rtcp_fb_nack;          /**< TX NACK state.     */
     int                      rtcp_fb_nack_cap_idx;  /**< RX NACK cap idx.   */
+
+    /* Media synchronization */
+    pjmedia_av_sync         *av_sync;               /**< Media sync.        */
+    pjmedia_av_sync_media   *av_sync_media;         /**< Media sync media   */
+
 } pjmedia_stream_common;
 
 
@@ -249,7 +255,6 @@ pjmedia_stream_common_send_rtcp_bye( pjmedia_stream_common *stream );
  * and generally it is not advisable for app to modify them.
  *
  * @param stream        The media stream.
- *
  * @param session_info  The stream session info.
  *
  * @return              PJ_SUCCESS on success.
@@ -259,6 +264,24 @@ pjmedia_stream_common_get_rtp_session_info(pjmedia_stream_common *stream,
                                    pjmedia_stream_rtp_sess_info *session_info);
 
 
+/**
+ * Set or reset media presentation synchronizer. The synchronizer manages
+ * presentation time of media streams in the session, e.g: audio & video.
+ *
+ * Application creates a media synchronizer and assign it to all media streams
+ * whose presentation time to be synchronized using this function.
+ *
+ * @param stream        The media stream.
+ * @param av_sync       The media presentation synchronizer, or NULL to
+ *                      remove this stream from current synchronizer.
+ *
+ * @return              PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t)
+pjmedia_stream_common_set_avsync(pjmedia_stream_common* stream,
+                                 pjmedia_av_sync* av_sync);
+
+
 /* Internal function. */
 
 /* Internal:  * Send RTCP SDES for the media stream. */