Implement audio video synchronization

pjmedia/build/Makefile

@@ -58,7 +58,7 @@ export _LDFLAGS := $(APP_THIRD_PARTY_LIBS) \
 #
 export PJMEDIA_SRCDIR = ../src/pjmedia
 export PJMEDIA_OBJS += $(OS_OBJS) $(M_OBJS) $(CC_OBJS) $(HOST_OBJS) \
-			alaw_ulaw.o alaw_ulaw_table.o avi_player.o \
+			alaw_ulaw.o alaw_ulaw_table.o avi_player.o av_sync.o \
 			bidirectional.o clock_thread.o codec.o conference.o \
 			conf_switch.o converter.o  converter_libswscale.o converter_libyuv.o \
 			delaybuf.o echo_common.o \

pjmedia/build/pjmedia.vcxproj

@@ -610,6 +610,7 @@
     <ClCompile Include="..\src\pjmedia\alaw_ulaw_table.c" />
     <ClCompile Include="..\src\pjmedia\audiodev.c" />
     <ClCompile Include="..\src\pjmedia\avi_player.c" />
+    <ClCompile Include="..\src\pjmedia\av_sync.c" />
     <ClCompile Include="..\src\pjmedia\bidirectional.c" />
     <ClCompile Include="..\src\pjmedia\clock_thread.c" />
     <ClCompile Include="..\src\pjmedia\codec.c" />
@@ -738,6 +739,7 @@
     <ClInclude Include="..\include\pjmedia\audiodev.h" />
     <ClInclude Include="..\include\pjmedia\avi.h" />
     <ClInclude Include="..\include\pjmedia\avi_stream.h" />
+    <ClInclude Include="..\include\pjmedia\av_sync.h" />
     <ClInclude Include="..\include\pjmedia\bidirectional.h" />
     <ClInclude Include="..\include\pjmedia\circbuf.h" />
     <ClInclude Include="..\include\pjmedia\clock.h" />

pjmedia/build/pjmedia.vcxproj.filters

@@ -230,6 +230,9 @@
     <ClCompile Include="..\src\pjmedia\echo_webrtc_aec3.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\src\pjmedia\av_sync.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\include\pjmedia\alaw_ulaw.h">
@@ -418,5 +421,8 @@
     <ClInclude Include="..\include\pjmedia\vid_conf.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\include\pjmedia\av_sync.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

pjmedia/include/pjmedia/av_sync.h

@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2025 Teluu Inc. (http://www.teluu.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+#ifndef __PJMEDIA_AV_SYNC_H__
+#define __PJMEDIA_AV_SYNC_H__
+
+ /**
+  * @file av_sync.h
+  * @brief Media Presentation Synchronization.
+  */
+#include <pjmedia/types.h>
+
+
+PJ_BEGIN_DECL
+
+
+/**
+ * @defgroup PJMEDIA_AV_SYNC Media Presentation Synchronization
+ * @ingroup PJMEDIA_PORT_CLOCK
+ * @brief Synchronize media presentation
+ * @{
+ */
+
+
+/*
+ * How it works in general:
+ * 1. App creates AV sync using pjmedia_av_sync_create().
+ * 2. App adds all media to be synchronized using pjmedia_av_sync_add_media().
+ * 3. Each time a media receives an RTCP-SR, update AV sync using
+ *    pjmedia_av_sync_update_ref().
+ * 4. Each time a media returns a frame to be rendered, e.g: via
+ *    port.get_frame(), update AV sync using pjmedia_av_sync_update_pts(),
+ *    the function will return a number:
+ *    - zero    : nothing to do,
+ *    - positive: increase delay as many as the returned number,
+ *    - negative: decrease delay as many as the returned number.
+ *
+ * Logic in pjmedia_av_sync_update_pts():
+ * - Calculate the absolute/NTP timestamp of the frame playback based on
+     the reference NTP and the supplied frame RTP timestamp. This timestamp
+     is usually called presentation time (pts).
+ * - If pts is the largest or most recent, set it as AV sync's max_ntp.
+ *   Otherwise, calculate the difference/delay of this pts to the max_ntp,
+ *   then request the media to speed up as much as the delay.
+ * - If after some requests the delay is still relatively high, request
+ *   the fastest media to slow down instead.
+ *
+ * Changes needed in stream:
+ * 1. Update ref whenever receive RTCP-SR.
+ * 2. Update presentation time whenever returns a frame (to play/render).
+ * 3. Add mechanism to calculate current delay & optimal delay from
+ *    jitter buffer size.
+ * 4. When AV sync requests to speed up, as long as the end delay is
+ *    not lower than the optimal delay, do it.
+ * 5. When AV sync requests to slow down, do it, i.e: increase and maintain
+ *    the jitter buffer size.
+ */
+
+
+/**
+ * AV sync, opaque.
+ */
+typedef struct pjmedia_av_sync pjmedia_av_sync;
+
+
+/**
+ * Media, opaque.
+ */
+typedef struct pjmedia_av_sync_media pjmedia_av_sync_media;
+
+
+/**
+ * Media setting.
+ */
+typedef struct {
+    char                       *name;
+    unsigned                    clock_rate;
+} pjmedia_av_sync_media_setting;
+
+
+/**
+ * Get default settings for media.
+ */
+PJ_DECL(void) pjmedia_av_sync_media_setting_default(
+                                pjmedia_av_sync_media_setting *setting);
+
+/**
+ * Create media synchronizer.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_create(
+                                pjmedia_endpt *endpt,
+                                const void *setting,
+                                pjmedia_av_sync **av_sync);
+
+
+/**
+ * Destroy media synchronizer.
+ */
+PJ_DECL(void) pjmedia_av_sync_destroy(pjmedia_av_sync* av_sync);
+
+
+/**
+ * Add media to synchronizer.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_add_media(
+                                pjmedia_av_sync* av_sync,
+                                const pjmedia_av_sync_media_setting* setting,
+                                pjmedia_av_sync_media** av_sync_media);
+
+
+/**
+ * Remove media from synchronizer.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_del_media(
+                                pjmedia_av_sync *av_sync,
+                                pjmedia_av_sync_media *av_sync_media);
+
+
+/**
+ * Update synchronizer about the last presentation timestamp of the specified
+ * media.
+ *
+ * Normally this function is called after a media renderer (video renderer or
+ * speaker) invokes get_frame() of the media source (e.g: audio/video stream).
+ *
+ * @return      0  : media is in sync, no action is needed.
+ *              <0 : speed up presentation by returned value milliseconds.
+ *              >0 : slow down presentation by returned value milliseconds.
+ */
+PJ_DECL(pj_int32_t) pjmedia_av_sync_update_pts(
+                                pjmedia_av_sync_media *av_sync_media,
+                                const pj_timestamp *pts);
+
+
+/**
+ * Update synchronizer about the last timestamp reference of the specified
+ * media.
+ *
+ * Normally this function is called after a media source receive RTCP-SR
+ * with info of an NTP timestamp correspoding to an RTP timestamp.
+ */
+PJ_DECL(pj_status_t) pjmedia_av_sync_update_ref(
+                                pjmedia_av_sync_media *av_sync_media,
+                                const pj_timestamp *ntp,
+                                const pj_timestamp *ts);
+
+
+
+/**
+ * @}
+ */
+
+
+PJ_END_DECL
+
+
+#endif  /* __PJMEDIA_AV_SYNC_H__ */

pjmedia/include/pjmedia/jbuf.h

@@ -102,6 +102,7 @@ typedef struct pjmedia_jb_state
     unsigned    min_prefetch;       /**< Minimum allowed prefetch, in frms. */
     unsigned    max_prefetch;       /**< Maximum allowed prefetch, in frms. */
     unsigned    max_count;          /**< Jitter buffer capacity, in frames. */
+    unsigned    min_delay_set;      /**< Minimum delay setting, in frames.  */
 
     /* Status */
     unsigned    burst;              /**< Current burst level, in frames     */
@@ -474,6 +475,20 @@ PJ_DECL(pj_status_t) pjmedia_jbuf_get_state( const pjmedia_jbuf *jb,
                                              pjmedia_jb_state *state );
 
 
+/**
+ * Set minimum delay of the jitter buffer. Normally jitter buffer tries to
+ * maintain the optimal delay calculated based on current burst level.
+ * When the minimum delay is set, the jitter buffer will adjust the delay to
+ * the greater of the optimal delay and the minimum delay.
+ *
+ * @param jb            The jitter buffer.
+ * @param min_delay     The minimum delay, in millisecond.
+ *
+ * @return              PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t) pjmedia_jbuf_set_min_delay(pjmedia_jbuf *jb,
+                                                unsigned min_delay);
+
 
 PJ_END_DECL
 

pjmedia/include/pjmedia/rtcp.h

@@ -263,6 +263,10 @@ typedef struct pjmedia_rtcp_session
 
     pj_uint32_t             rx_lsr;     /**< NTP ts in last SR received     */
     pj_timestamp            rx_lsr_time;/**< Time when last SR is received  */
+    pj_uint32_t             rx_lsr_ts;  /**< RTP ts in last SR received     */
+    pj_timestamp            rx_lsr_ntp; /**< Original/64bit NTP ts in last
+                                             SR received                    */
+
     pj_uint32_t             peer_ssrc;  /**< Peer SSRC                      */
 
     pjmedia_rtcp_stat       stat;       /**< Bidirectional stream stat.     */

pjmedia/include/pjmedia/stream_common.h

@@ -24,6 +24,7 @@
  * @brief Stream common functions.
  */
 
+#include <pjmedia/av_sync.h>
 #include <pjmedia/codec.h>
 #include <pjmedia/jbuf.h>
 #include <pjmedia/sdp.h>
@@ -151,6 +152,11 @@ typedef struct pjmedia_stream_common
     int                      pending_rtcp_fb_nack;  /**< Any pending NACK?  */
     pjmedia_rtcp_fb_nack     rtcp_fb_nack;          /**< TX NACK state.     */
     int                      rtcp_fb_nack_cap_idx;  /**< RX NACK cap idx.   */
+
+    /* Media synchronization */
+    pjmedia_av_sync         *av_sync;               /**< Media sync.        */
+    pjmedia_av_sync_media   *av_sync_media;         /**< Media sync media   */
+
 } pjmedia_stream_common;
 
 
@@ -249,7 +255,6 @@ pjmedia_stream_common_send_rtcp_bye( pjmedia_stream_common *stream );
  * and generally it is not advisable for app to modify them.
  *
  * @param stream        The media stream.
- *
  * @param session_info  The stream session info.
  *
  * @return              PJ_SUCCESS on success.
@@ -259,6 +264,24 @@ pjmedia_stream_common_get_rtp_session_info(pjmedia_stream_common *stream,
                                    pjmedia_stream_rtp_sess_info *session_info);
 
 
+/**
+ * Set or reset media presentation synchronizer. The synchronizer manages
+ * presentation time of media streams in the session, e.g: audio & video.
+ *
+ * Application creates a media synchronizer and assign it to all media streams
+ * whose presentation time to be synchronized using this function.
+ *
+ * @param stream        The media stream.
+ * @param av_sync       The media presentation synchronizer, or NULL to
+ *                      remove this stream from current synchronizer.
+ *
+ * @return              PJ_SUCCESS on success.
+ */
+PJ_DECL(pj_status_t)
+pjmedia_stream_common_set_avsync(pjmedia_stream_common* stream,
+                                 pjmedia_av_sync* av_sync);
+
+
 /* Internal function. */
 
 /* Internal:  * Send RTCP SDES for the media stream. */