diff --git a/pjmedia/build/Makefile b/pjmedia/build/Makefile index baa6b258a6..d9bf298454 100644 --- a/pjmedia/build/Makefile +++ b/pjmedia/build/Makefile @@ -58,7 +58,7 @@ export _LDFLAGS := $(APP_THIRD_PARTY_LIBS) \ # export PJMEDIA_SRCDIR = ../src/pjmedia export PJMEDIA_OBJS += $(OS_OBJS) $(M_OBJS) $(CC_OBJS) $(HOST_OBJS) \ - alaw_ulaw.o alaw_ulaw_table.o avi_player.o \ + alaw_ulaw.o alaw_ulaw_table.o avi_player.o av_sync.o \ bidirectional.o clock_thread.o codec.o conference.o \ conf_switch.o converter.o converter_libswscale.o converter_libyuv.o \ delaybuf.o echo_common.o \ diff --git a/pjmedia/build/pjmedia.vcproj b/pjmedia/build/pjmedia.vcproj index 28d0d2cb43..9aed93e0e8 100644 --- a/pjmedia/build/pjmedia.vcproj +++ b/pjmedia/build/pjmedia.vcproj @@ -3342,6 +3342,10 @@ RelativePath="..\src\pjmedia\audiodev.c" > + + @@ -7495,6 +7499,10 @@ RelativePath="..\include\pjmedia\audiodev.h" > + + diff --git a/pjmedia/build/pjmedia.vcxproj b/pjmedia/build/pjmedia.vcxproj index 146bfa8ded..be1379087d 100644 --- a/pjmedia/build/pjmedia.vcxproj +++ b/pjmedia/build/pjmedia.vcxproj @@ -610,6 +610,7 @@ + @@ -738,6 +739,7 @@ + diff --git a/pjmedia/build/pjmedia.vcxproj.filters b/pjmedia/build/pjmedia.vcxproj.filters index 222e58cb14..2a81997973 100644 --- a/pjmedia/build/pjmedia.vcxproj.filters +++ b/pjmedia/build/pjmedia.vcxproj.filters @@ -230,6 +230,9 @@ Source Files + + Source Files + @@ -418,5 +421,8 @@ Header Files + + Header Files + \ No newline at end of file diff --git a/pjmedia/include/pjmedia/av_sync.h b/pjmedia/include/pjmedia/av_sync.h new file mode 100644 index 0000000000..5324a70f36 --- /dev/null +++ b/pjmedia/include/pjmedia/av_sync.h @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2025 Teluu Inc. (http://www.teluu.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#ifndef __PJMEDIA_AV_SYNC_H__ +#define __PJMEDIA_AV_SYNC_H__ + + /** + * @file av_sync.h + * @brief Inter-media Synchronization. + */ +#include + + +PJ_BEGIN_DECL + + +/** + * @defgroup PJMEDIA_AV_SYNC Inter-media Synchronization + * @ingroup PJMEDIA_SESSION + * @brief Synchronize presentation time of multiple media in a session. + * @{ + * + * A call session may consist of multiple media, e.g: some audio and some + * video, which frequently have different delays when presented in the + * receiver side. This module synchronizes all media in the same session + * based on NTP timestamp & RTP timestamp info provided by the sender in + * RTCP SR. + * + * Here are steps to use this module: + * 1. Create AV sync using #pjmedia_av_sync_create(). + * 2. Adds all media to be synchronized using #pjmedia_av_sync_add_media(). + * 3. Call #pjmedia_av_sync_update_ref() each time the media receiving + * an RTCP SR packet. + * 4. Call #pjmedia_av_sync_update_pts() each time the media returning + * a frame to be presented, e.g: via port.get_frame(). The function may + * request the media to adjust its delay. + * 5. Call #pjmedia_av_sync_del_media() when a media is removed from the + * session. + * 6. Call #pjmedia_av_sync_destroy() when the session is ended. + * + * The primary synchronization logic is implemented within the + * #pjmedia_av_sync_update_pts() function. This function will calculate + * the lag between the calling media to the earliest media and will provide + * a feedback to the calling media whether it is in synchronized state, + * late, or early so the media can respond accordingly. + * Initially this function will try to request slower media to speed up. + * If after a specific number of requests (i.e: configurable via + * PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT) and the lag is still beyond a tolerable + * value (i.e: configurable via PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC), the + * function will issue slow down request to the fastest media. + */ + + +/** + * Inter-media synchronizer, opaque. + */ +typedef struct pjmedia_av_sync pjmedia_av_sync; + + +/** + * Media synchronization handle, opaque. + */ +typedef struct pjmedia_av_sync_media pjmedia_av_sync_media; + + +/** + * Synchronizer settings. + */ +typedef struct { + /** + * Name of the syncrhonizer + */ + char *name; + + /** + * Streaming mode. If set to PJ_TRUE, the delay adjustment values will + * be smoothened and marked up to prevent possible delay increase on + * all media. + */ + pj_bool_t is_streaming; + +} pjmedia_av_sync_setting; + + +/** + * Media settings. + */ +typedef struct { + /** + * Name of the media + */ + char *name; + + /** + * Media type. + */ + pjmedia_type type; + + /** + * Media clock rate or sampling rate. + */ + unsigned clock_rate; + +} pjmedia_av_sync_media_setting; + + +/** + * Get default settings for synchronizer. + * + * @param setting The synchronizer settings. + */ +PJ_DECL(void) pjmedia_av_sync_setting_default( + pjmedia_av_sync_setting *setting); + +/** + * Get default settings for media. + * + * @param setting The media settings. + */ +PJ_DECL(void) pjmedia_av_sync_media_setting_default( + pjmedia_av_sync_media_setting *setting); + +/** + * Create media synchronizer. + * + * @param pool The memory pool. + * @param option The synchronizer settings. + * @param av_sync The pointer to receive the media synchronizer. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_status_t) pjmedia_av_sync_create( + pj_pool_t *pool, + const pjmedia_av_sync_setting *setting, + pjmedia_av_sync **av_sync); + + +/** + * Destroy media synchronizer. + * + * @param av_sync The media synchronizer. + */ +PJ_DECL(void) pjmedia_av_sync_destroy(pjmedia_av_sync *av_sync); + + +/** + * Reset synchronization states. Any existing media will NOT be removed, + * but their states will be reset. + * + * @param av_sync The media synchronizer. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_status_t) pjmedia_av_sync_reset(pjmedia_av_sync *av_sync); + + +/** + * Add a media to synchronizer. + * + * @param av_sync The media synchronizer. + * @param setting The media settings. + * @param av_sync_media The pointer to receive the media synchronization + * handle. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_status_t) pjmedia_av_sync_add_media( + pjmedia_av_sync* av_sync, + const pjmedia_av_sync_media_setting *setting, + pjmedia_av_sync_media **av_sync_media); + + +/** + * Remove a media from synchronizer. + * + * @param av_sync The media synchronizer. + * @param av_sync_media The media synchronization handle. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_status_t) pjmedia_av_sync_del_media( + pjmedia_av_sync *av_sync, + pjmedia_av_sync_media *av_sync_media); + + +/** + * Update synchronizer about the last presentation timestamp of the specified + * media. Normally this function is called each time the media produces + * a frame to be rendered (e.g: in port's get_frame() method). Upon returning, + * the media may be requested to adjust its delay so it matches to the + * earliest or the latest media, i.e: by speeding up or slowing down. + * + * Initially this function will try to request slower media to speed up. + * If after a specific number of requests (i.e: configurable via + * PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT) and the lag is still beyond a tolerable + * value (i.e: configurable via PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC), the + * function will issue slow down request to the fastest media. + * + * @param av_sync_media The media synchronization handle. + * @param pts The presentation timestamp. + * @param adjust_delay Optional pointer to receive adjustment delay + * required, in milliseconds, to make this media + * synchronized to the fastest media. + * Possible output values are: + * 0 when no action is needed, + * possitive value when increasing delay is needed, + * or negative value when decreasing delay is needed. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_status_t) pjmedia_av_sync_update_pts( + pjmedia_av_sync_media *av_sync_media, + const pj_timestamp *pts, + pj_int32_t *adjust_delay); + + +/** + * Update synchronizer about reference timestamps of the specified media. + * Normally this function is called each time the media receives RTCP SR + * packet. + * + * @param av_sync_media The media synchronization handle. + * @param ntp The NTP timestamp info from RTCP SR. + * @param ts The RTP timestamp info from RTCP SR. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_status_t) pjmedia_av_sync_update_ref( + pjmedia_av_sync_media *av_sync_media, + const pj_timestamp *ntp, + const pj_timestamp *ts); + + +/** + * @} + */ + + +PJ_END_DECL + + +#endif /* __PJMEDIA_AV_SYNC_H__ */ diff --git a/pjmedia/include/pjmedia/avi_stream.h b/pjmedia/include/pjmedia/avi_stream.h index 17839b469e..ada43eb77f 100644 --- a/pjmedia/include/pjmedia/avi_stream.h +++ b/pjmedia/include/pjmedia/avi_stream.h @@ -45,7 +45,13 @@ enum pjmedia_avi_file_player_option * Tell the file player to return NULL frame when the whole * file has been played. */ - PJMEDIA_AVI_FILE_NO_LOOP = 1 + PJMEDIA_AVI_FILE_NO_LOOP = 1, + + /** + * Set the file player to permit independent playback of audio and + * video streams without synchronization. + */ + PJMEDIA_AVI_FILE_NO_SYNC = 2 }; /** @@ -64,9 +70,16 @@ typedef struct pjmedia_avi_streams pjmedia_avi_streams; * reading AVI file with uncompressed video format and * 16 bit PCM or compressed G.711 A-law/U-law audio format. * + * By default, avi streams will loop the file playback and synchronize + * audio and video streams. To change this behavior, use the flags parameter. + * + * When synchronization is enabled, the file player will wait for all + * media streams to reach the end of file before rewinding the file. + * * @param pool Pool to create the streams. * @param filename File name to open. - * @param flags Avi streams creation flags. + * @param flags Avi streams creation flags, bitmask combination of + * #pjmedia_avi_file_player_option. * @param p_streams Pointer to receive the avi streams instance. * * @return PJ_SUCCESS on success. diff --git a/pjmedia/include/pjmedia/config.h b/pjmedia/include/pjmedia/config.h index f75029aa0a..96fbe639a1 100644 --- a/pjmedia/include/pjmedia/config.h +++ b/pjmedia/include/pjmedia/config.h @@ -1734,6 +1734,32 @@ #undef PJMEDIA_VID_STREAM_CHECK_RTP_PT #define PJMEDIA_VID_STREAM_CHECK_RTP_PT PJMEDIA_STREAM_CHECK_RTP_PT + +/** + * Maximum tolerable presentation lag from the earliest to the latest media, + * in milliseconds, in inter-media synchronization. When the delay is + * higher than this setting, the media synchronizer will request the slower + * media to speed up. And if after a number of speed up requests the delay + * is still beyond this setting, the fastest media will be requested to + * slow down. + * + * Default: 45 ms + */ +#ifndef PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC +# define PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC 45 +#endif + + +/** + * Maximum number of speed up request to synchronize presentation time, + * before a slow down request to the fastest media is issued. + * + * Default: 10 + */ +#ifndef PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT +# define PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT 10 +#endif + /** * @} */ diff --git a/pjmedia/include/pjmedia/jbuf.h b/pjmedia/include/pjmedia/jbuf.h index 6f727494de..9f09af051e 100644 --- a/pjmedia/include/pjmedia/jbuf.h +++ b/pjmedia/include/pjmedia/jbuf.h @@ -102,6 +102,7 @@ typedef struct pjmedia_jb_state unsigned min_prefetch; /**< Minimum allowed prefetch, in frms. */ unsigned max_prefetch; /**< Maximum allowed prefetch, in frms. */ unsigned max_count; /**< Jitter buffer capacity, in frames. */ + unsigned min_delay_set; /**< Minimum delay setting, in frames. */ /* Status */ unsigned burst; /**< Current burst level, in frames */ @@ -474,6 +475,20 @@ PJ_DECL(pj_status_t) pjmedia_jbuf_get_state( const pjmedia_jbuf *jb, pjmedia_jb_state *state ); +/** + * Set minimum delay of the jitter buffer. Normally jitter buffer tries to + * maintain the optimal delay calculated based on current burst level. + * When the minimum delay is set, the jitter buffer will adjust the delay to + * the greater of the optimal delay and the minimum delay. + * + * @param jb The jitter buffer. + * @param min_delay The minimum delay, in millisecond. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_status_t) pjmedia_jbuf_set_min_delay(pjmedia_jbuf *jb, + unsigned min_delay); + PJ_END_DECL diff --git a/pjmedia/include/pjmedia/rtcp.h b/pjmedia/include/pjmedia/rtcp.h index d9ab062c45..5e9897fa24 100644 --- a/pjmedia/include/pjmedia/rtcp.h +++ b/pjmedia/include/pjmedia/rtcp.h @@ -263,6 +263,10 @@ typedef struct pjmedia_rtcp_session pj_uint32_t rx_lsr; /**< NTP ts in last SR received */ pj_timestamp rx_lsr_time;/**< Time when last SR is received */ + pj_uint32_t rx_lsr_ts; /**< RTP ts in last SR received */ + pj_timestamp rx_lsr_ntp; /**< Original/64bit NTP ts in last + SR received */ + pj_uint32_t peer_ssrc; /**< Peer SSRC */ pjmedia_rtcp_stat stat; /**< Bidirectional stream stat. */ diff --git a/pjmedia/include/pjmedia/stream_common.h b/pjmedia/include/pjmedia/stream_common.h index 5b4c0da312..d077fdf3c2 100644 --- a/pjmedia/include/pjmedia/stream_common.h +++ b/pjmedia/include/pjmedia/stream_common.h @@ -24,6 +24,7 @@ * @brief Stream common functions. */ +#include #include #include #include @@ -151,6 +152,11 @@ typedef struct pjmedia_stream_common int pending_rtcp_fb_nack; /**< Any pending NACK? */ pjmedia_rtcp_fb_nack rtcp_fb_nack; /**< TX NACK state. */ int rtcp_fb_nack_cap_idx; /**< RX NACK cap idx. */ + + /* Media synchronization */ + pjmedia_av_sync *av_sync; /**< Media sync. */ + pjmedia_av_sync_media *av_sync_media; /**< Media sync media */ + } pjmedia_stream_common; @@ -249,7 +255,6 @@ pjmedia_stream_common_send_rtcp_bye( pjmedia_stream_common *stream ); * and generally it is not advisable for app to modify them. * * @param stream The media stream. - * * @param session_info The stream session info. * * @return PJ_SUCCESS on success. @@ -259,6 +264,24 @@ pjmedia_stream_common_get_rtp_session_info(pjmedia_stream_common *stream, pjmedia_stream_rtp_sess_info *session_info); +/** + * Set or reset media presentation synchronizer. The synchronizer manages + * presentation time of media streams in the session, e.g: audio & video. + * + * Application creates a media synchronizer and assign it to all media streams + * whose presentation time to be synchronized using this function. + * + * @param stream The media stream. + * @param av_sync The media presentation synchronizer, or NULL to + * remove this stream from current synchronizer. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_status_t) +pjmedia_stream_common_set_avsync(pjmedia_stream_common* stream, + pjmedia_av_sync* av_sync); + + /* Internal function. */ /* Internal: * Send RTCP SDES for the media stream. */ diff --git a/pjmedia/src/pjmedia/av_sync.c b/pjmedia/src/pjmedia/av_sync.c new file mode 100644 index 0000000000..30dff51559 --- /dev/null +++ b/pjmedia/src/pjmedia/av_sync.c @@ -0,0 +1,469 @@ +/* + * Copyright (C) 2025 Teluu Inc. (http://www.teluu.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include + +/* Enable/disable trace */ +#if 0 +# define TRACE_(x) PJ_LOG(5, x) +#else +# define TRACE_(x) +#endif + +/* AV sync media */ +struct pjmedia_av_sync_media +{ + PJ_DECL_LIST_MEMBER(struct pjmedia_av_sync_media); + + pjmedia_av_sync *av_sync; /* The AV sync instance */ + pjmedia_av_sync_media_setting setting; /* Media settings */ + + /* Reference timestamp */ + pj_bool_t is_ref_set; /* Has reference been set? */ + pj_timestamp ref_ts; /* Ref ts, in sample units */ + pj_timestamp ref_ntp; /* Ref ts, in NTP units */ + + /* Last presentation timestamp */ + pj_timestamp last_ntp; /* Last PTS, in NTP units */ + pj_int32_t smooth_diff; + + /* Delay adjustment requested to this media */ + pj_int32_t last_adj_delay_req; /* Last requested delay */ + unsigned adj_delay_req_cnt; /* Request counter */ +}; + + +/* AV sync */ +struct pjmedia_av_sync +{ + pj_pool_t *pool; + pjmedia_av_sync_media media_list; + pjmedia_av_sync_media free_media_list; + pj_grp_lock_t *grp_lock; + unsigned last_idx; + pjmedia_av_sync_setting setting; + + /* Maximum NTP time of all media */ + pj_timestamp max_ntp; + + /* Some media cannot catch up, request for slow down, in milliseconds */ + unsigned slowdown_req_ms; +}; + + +static void ntp_add_ts(pj_timestamp* ntp, unsigned ts, unsigned clock_rate) +{ + pj_timestamp ts_diff; + + ts_diff.u64 = ((pj_uint64_t)ts << 32) / clock_rate; + pj_add_timestamp(ntp, &ts_diff); +} + +static unsigned ntp_to_ms(const pj_timestamp* ntp) +{ + pj_uint64_t ms; + + ms = ntp->u32.hi * 1000 + (((pj_uint64_t)ntp->u32.lo * 1000) >> 32); + return (unsigned)ms; +} + + +/* AV sync destroy handler. */ +static void avs_on_destroy(void* arg) +{ + pjmedia_av_sync* avs = (pjmedia_av_sync*)arg; + PJ_LOG(4, (avs->setting.name, "%s destroyed", avs->setting.name)); + pj_pool_release(avs->pool); +} + + +/* Get default values for synchronizer settings. */ +PJ_DEF(void) pjmedia_av_sync_setting_default( + pjmedia_av_sync_setting *setting) +{ + pj_bzero(setting, sizeof(*setting)); +} + + +/* Get default values for media settings. */ +PJ_DEF(void) pjmedia_av_sync_media_setting_default( + pjmedia_av_sync_media_setting* setting) +{ + pj_bzero(setting, sizeof(*setting)); +} + + +/* Create media synchronizer. */ +PJ_DEF(pj_status_t) pjmedia_av_sync_create( + pj_pool_t *pool_, + const pjmedia_av_sync_setting *setting, + pjmedia_av_sync **av_sync) +{ + pj_pool_t* pool = NULL; + pjmedia_av_sync* avs = NULL; + pj_status_t status; + + PJ_ASSERT_RETURN(pool_ && av_sync && setting, PJ_EINVAL); + + pool = pj_pool_create(pool_->factory, "avsync%p", 512, 512, NULL); + if (!pool) { + status = PJ_ENOMEM; + goto on_error; + } + + avs = PJ_POOL_ZALLOC_T(pool, pjmedia_av_sync); + if (!avs) { + status = PJ_ENOMEM; + goto on_error; + } + avs->pool = pool; + avs->setting = *setting; + if (setting->name) { + pj_size_t len = PJ_MIN(PJ_MAX_OBJ_NAME, + pj_ansi_strlen(setting->name)+1); + avs->setting.name = pj_pool_zalloc(avs->pool, len); + pj_ansi_snprintf(avs->setting.name, len, "%s", setting->name); + } else { + avs->setting.name = pool->obj_name; + } + + status = pj_grp_lock_create_w_handler(pool, NULL, avs, &avs_on_destroy, + &avs->grp_lock); + if (status != PJ_SUCCESS) + goto on_error; + + pj_grp_lock_add_ref(avs->grp_lock); + pj_list_init(&avs->media_list); + pj_list_init(&avs->free_media_list); + + PJ_LOG(4, (avs->setting.name, "%s created", avs->setting.name)); + *av_sync = avs; + return PJ_SUCCESS; + +on_error: + if (pool) + pj_pool_release(pool); + + return status; +} + + +/* Destroy media synchronizer. */ +PJ_DEF(void) pjmedia_av_sync_destroy(pjmedia_av_sync* avs) +{ + PJ_ASSERT_ON_FAIL(avs, return); + PJ_LOG(4, (avs->setting.name, "%s destroy requested", + avs->setting.name)); + pj_grp_lock_dec_ref(avs->grp_lock); +} + + +/* Reset synchronization states. */ +PJ_DEF(pj_status_t) pjmedia_av_sync_reset(pjmedia_av_sync *avs) +{ + pjmedia_av_sync_media* m; + PJ_ASSERT_RETURN(avs, PJ_EINVAL); + + pj_grp_lock_acquire(avs->grp_lock); + avs->max_ntp.u64 = 0; + avs->slowdown_req_ms = 0; + + m = avs->media_list.next; + while (m != &avs->media_list) { + m->is_ref_set = PJ_FALSE; + m->last_ntp.u64 = 0; + m->last_adj_delay_req = 0; + m->adj_delay_req_cnt = 0; + m->smooth_diff = 0; + m = m->next; + } + pj_grp_lock_release(avs->grp_lock); + return PJ_SUCCESS; +} + + +/* Add media to synchronizer. */ +PJ_DEF(pj_status_t) pjmedia_av_sync_add_media( + pjmedia_av_sync *avs, + const pjmedia_av_sync_media_setting *setting, + pjmedia_av_sync_media **media) +{ + pjmedia_av_sync_media* m; + pj_status_t status = PJ_SUCCESS; + char* m_name; + + PJ_ASSERT_RETURN(avs && media && setting, PJ_EINVAL); + + pj_grp_lock_acquire(avs->grp_lock); + + /* Get media from free list, if any, otherwise allocate a new one */ + if (!pj_list_empty(&avs->free_media_list)) { + m = avs->free_media_list.next; + pj_list_erase(m); + m_name = m->setting.name; + } else { + m = PJ_POOL_ZALLOC_T(avs->pool, pjmedia_av_sync_media); + m_name = pj_pool_zalloc(avs->pool, PJ_MAX_OBJ_NAME); + if (!m || !m_name) { + status = PJ_ENOMEM; + goto on_return; + } + } + + m->av_sync = avs; + m->setting = *setting; + if (setting->name) { + pj_ansi_snprintf(m_name, PJ_MAX_OBJ_NAME, "%s", setting->name); + } else { + pj_ansi_snprintf(m_name, PJ_MAX_OBJ_NAME, "avs_med_%d", + ++avs->last_idx); + } + m->setting.name = m_name; + + pj_list_push_back(&avs->media_list, m); + pj_grp_lock_add_ref(avs->grp_lock); + + *media = m; + PJ_LOG(4, (avs->setting.name, "Added media %s, clock rate=%d", + m->setting.name, m->setting.clock_rate)); + +on_return: + pj_grp_lock_release(avs->grp_lock); + return status; +} + + +/* Remove media from synchronizer. */ +PJ_DEF(pj_status_t) pjmedia_av_sync_del_media( + pjmedia_av_sync *avs, + pjmedia_av_sync_media *media) +{ + PJ_ASSERT_RETURN(media, PJ_EINVAL); + PJ_ASSERT_RETURN(!avs || media->av_sync == avs, PJ_EINVAL); + + if (!avs) + avs = media->av_sync; + + pj_grp_lock_acquire(avs->grp_lock); + pj_list_erase(media); + + /* Zero some fields */ + media->is_ref_set = PJ_FALSE; + media->last_adj_delay_req = 0; + media->adj_delay_req_cnt = 0; + media->smooth_diff = 0; + + pj_list_push_back(&avs->free_media_list, media); + pj_grp_lock_release(avs->grp_lock); + + PJ_LOG(4, (avs->setting.name, "Removed media %s", media->setting.name)); + pj_grp_lock_dec_ref(avs->grp_lock); + + return PJ_SUCCESS; +} + + +/* Update synchronizer about the last timestamp reference of the specified + * media. + */ +PJ_DEF(pj_status_t) pjmedia_av_sync_update_ref( + pjmedia_av_sync_media* media, + const pj_timestamp* ntp, + const pj_timestamp* ts) +{ + PJ_ASSERT_RETURN(media && ntp && ts, PJ_EINVAL); + + media->ref_ntp = *ntp; + media->ref_ts = *ts; + media->is_ref_set = PJ_TRUE; + TRACE_((media->av_sync->setting.name, "%s updates ref ntp=%u ts=%u", + media->setting.name, ntp->u64, ts->u64)); + + return PJ_SUCCESS; +} + + +PJ_DEF(pj_status_t) pjmedia_av_sync_update_pts( + pjmedia_av_sync_media *media, + const pj_timestamp *pts, + pj_int32_t *adjust_delay) +{ + pjmedia_av_sync *avs; + pj_int32_t diff; + pj_timestamp max_ntp; + + PJ_ASSERT_RETURN(media && media->av_sync && pts, PJ_EINVAL); + + /* Reset the adjustment delay */ + if (adjust_delay) + *adjust_delay = 0; + + /* Make sure we have a reference */ + if (!media->is_ref_set) + return PJ_EINVALIDOP; + + diff = pj_timestamp_diff32(&media->ref_ts, pts); + + /* Only process if pts is increasing */ + if (diff <= 0) + return PJ_ETOOSMALL; + + avs = media->av_sync; + TRACE_((avs->setting.name, "%s updates pts=%u", + media->setting.name, pts->u64)); + + /* Update last presentation time */ + media->last_ntp = media->ref_ntp; + ntp_add_ts(&media->last_ntp, diff, media->setting.clock_rate); + + /* Get NTP timestamp of the earliest media */ + pj_grp_lock_acquire(avs->grp_lock); + max_ntp = avs->max_ntp; + pj_grp_lock_release(avs->grp_lock); + + /* Check if this media is the fastest/earliest */ + if (pj_cmp_timestamp(&media->last_ntp, &max_ntp) > 0) { + /* Yes, it is the fastest, update the max timestamp */ + pj_grp_lock_acquire(avs->grp_lock); + avs->max_ntp = media->last_ntp; + pj_grp_lock_release(avs->grp_lock); + + /* Check if there is any request to slow down */ + if (avs->slowdown_req_ms) { + media->last_adj_delay_req = avs->slowdown_req_ms; + media->adj_delay_req_cnt = 0; + avs->slowdown_req_ms = 0; + TRACE_((avs->setting.name, + "%s is requested to slow down by %dms", + media->setting.name, media->last_adj_delay_req)); + if (adjust_delay) + *adjust_delay = media->last_adj_delay_req; + + return PJ_SUCCESS; + } + } else { + /* Not the fastest. */ + pj_timestamp ntp_diff = max_ntp; + unsigned ms_diff, ms_req; + + /* First, check the lag from the fastest. */ + pj_sub_timestamp(&ntp_diff, &media->last_ntp); + ms_diff = ntp_to_ms(&ntp_diff); + + /* For streaming, smoothen (apply weight of 9 for current lag), + * and round down the lag to the nearest 10. + */ + if (avs->setting.is_streaming) { + ms_diff = ((ms_diff + 9 * media->smooth_diff) / 100) * 10; + media->smooth_diff = ms_diff; + } + + /* The lag is tolerable, just return 0 */ + if (ms_diff <= PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC) { + if (media->last_adj_delay_req) { + TRACE_((avs->setting.name, + "%s lag looks good now=%ums", + media->setting.name, ms_diff)); + } + /* Reset the request delay & counter */ + media->adj_delay_req_cnt = 0; + media->last_adj_delay_req = 0; + + return PJ_SUCCESS; + } + + /* Check if any speed-up request has been done before */ + if (media->last_adj_delay_req) { + /* Check if request number has reached limit */ + if (media->adj_delay_req_cnt>=PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT) + { + + /* After several requests this media still cannot catch up, + * signal the synchronizer to slow down the fastest media. + * + * For streaming mode, request slow down 3/4 of required to + * prevent possible delay increase on all media. + */ + ms_req = ms_diff; + if (avs->setting.is_streaming) + ms_req = ms_req * 3/4; + + if (avs->slowdown_req_ms < ms_req) + avs->slowdown_req_ms = ms_req; + + TRACE_((avs->setting.name, + "%s request limit has been reached, requesting " + "the fastest media to slow down by %ums", + media->setting.name, avs->slowdown_req_ms)); + + /* Reset the request counter. + * And still keep requesting for speed up, shouldn't we? + */ + media->adj_delay_req_cnt = 0; + } else { + pj_int32_t progress, min_expected; + + /* Check if the previous delay request has shown some + * progress. + */ + progress = (-media->last_adj_delay_req) - ms_diff; + min_expected = -media->last_adj_delay_req / + (PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT - + media->adj_delay_req_cnt + 1); + if (progress >= min_expected) { + /* Yes, let's just request again and wait */ + TRACE_((avs->setting.name, + "%s speeds up in progress, current lag=%ums", + media->setting.name, ms_diff)); + } + } + } else { + /* First request to speed up */ + media->adj_delay_req_cnt = 0; + } + + /* Request the media to speed up & increment the counter. + * + * For streaming mode, request speed-up 4/3 of required to + * prevent possible delay increase on all media. + */ + ms_req = ms_diff; + if (avs->setting.is_streaming) + ms_req = ms_req * 4/3; + + media->last_adj_delay_req = -(pj_int32_t)ms_req; + media->adj_delay_req_cnt++; + + TRACE_((avs->setting.name, + "%s is requested to speed up #%d by %dms", + media->setting.name, media->adj_delay_req_cnt, + -media->last_adj_delay_req)); + + if (adjust_delay) + *adjust_delay = media->last_adj_delay_req; + + return PJ_SUCCESS; + } + + return PJ_SUCCESS; +} diff --git a/pjmedia/src/pjmedia/avi_player.c b/pjmedia/src/pjmedia/avi_player.c index f0fe2e9922..1ea599b10e 100644 --- a/pjmedia/src/pjmedia/avi_player.c +++ b/pjmedia/src/pjmedia/avi_player.c @@ -21,10 +21,12 @@ */ #include #include +#include #include #include #include #include +#include #include #include #include @@ -117,9 +119,13 @@ static avi_fmt_info avi_fmts[] = struct pjmedia_avi_streams { - pj_pool_t *pool; - unsigned num_streams; - pjmedia_port **streams; + pj_pool_t *pool; + unsigned num_streams; + pjmedia_port **streams; + + /* AV synchronization */ + pjmedia_av_sync *avsync; + pj_size_t eof_cnt; }; struct avi_reader_port @@ -128,7 +134,6 @@ struct avi_reader_port unsigned stream_id; unsigned options; pjmedia_format_id fmt_id; - unsigned usec_per_frame; pj_uint16_t bits_per_sample; pj_bool_t eof; pj_off_t fsize; @@ -136,8 +141,15 @@ struct avi_reader_port pj_uint8_t pad; pj_oshandle_t fd; pj_ssize_t size_left; + + pj_size_t frame_cnt; pj_timestamp next_ts; + /* AV synchronization */ + pjmedia_av_sync_media *avsync_media; + pj_size_t slow_down_frm; + pjmedia_avi_streams *avi_streams; + pj_status_t (*cb)(pjmedia_port*, void*); pj_bool_t subscribed; void (*cb2)(pjmedia_port*, void*); @@ -203,10 +215,27 @@ static pj_status_t file_read3(pj_oshandle_t fd, void *data, pj_ssize_t size, static void streams_on_destroy(void *arg) { pjmedia_avi_streams *streams = (pjmedia_avi_streams*)arg; + + if (streams->avsync) + pjmedia_av_sync_destroy(streams->avsync); pj_pool_safe_release(&streams->pool); } +/* Get filename from path */ +static const char *get_fname(const char *path) +{ + pj_size_t len = pj_ansi_strlen(path); + const char *p = path + len - 1; + + while (p > path) { + if (*p == '\\' || *p == '/' || *p == ':') + return p + 1; + --p; + } + return p; +} + /* * Create AVI player port. */ @@ -495,6 +524,7 @@ pjmedia_avi_player_create_streams(pj_pool_t *pool_, for (i = 0; i < nstr; i++) { strl_hdr_t *strl_hdr = &avi_hdr.strl_hdr[fport[i]->stream_id]; + char port_name[PJ_MAX_OBJ_NAME]; /* Initialize */ fport[i]->options = options; @@ -512,7 +542,7 @@ pjmedia_avi_player_create_streams(pj_pool_t *pool_, strl_hdr->codec); fport[i]->bits_per_sample = (vfi ? vfi->bpp : 0); - fport[i]->usec_per_frame = avi_hdr.avih_hdr.usec_per_frame; + //fport[i]->usec_per_frame = avi_hdr.avih_hdr.usec_per_frame; pjmedia_format_init_video(&fport[i]->base.info.fmt, fport[i]->fmt_id, strf_hdr->biWidth, @@ -540,7 +570,7 @@ pjmedia_avi_player_create_streams(pj_pool_t *pool_, &avi_hdr.strf_hdr[fport[i]->stream_id].strf_audio_hdr; fport[i]->bits_per_sample = strf_hdr->bits_per_sample; - fport[i]->usec_per_frame = avi_hdr.avih_hdr.usec_per_frame; + //fport[i]->usec_per_frame = avi_hdr.avih_hdr.usec_per_frame; pjmedia_format_init_audio(&fport[i]->base.info.fmt, fport[i]->fmt_id, strf_hdr->sample_rate, @@ -559,17 +589,61 @@ pjmedia_avi_player_create_streams(pj_pool_t *pool_, } } - pj_strdup2(pool, &fport[i]->base.info.name, filename); + pj_ansi_snprintf(port_name, sizeof(port_name), "%s-of-%s", + pjmedia_type_name(fport[i]->base.info.fmt.type), + get_fname(filename)); + pj_strdup2(pool, &fport[i]->base.info.name, port_name); } /* Done. */ - *p_streams = pj_pool_alloc(pool, sizeof(pjmedia_avi_streams)); + *p_streams = pj_pool_calloc(pool, 1, sizeof(pjmedia_avi_streams)); (*p_streams)->num_streams = nstr; (*p_streams)->streams = pj_pool_calloc(pool, (*p_streams)->num_streams, sizeof(pjmedia_port *)); for (i = 0; i < nstr; i++) (*p_streams)->streams[i] = &fport[i]->base; + /* Create AV synchronizer, if not disabled */ + if ((options & PJMEDIA_AVI_FILE_NO_SYNC) == 0) { + pjmedia_av_sync *avsync; + pj_timestamp ts_zero = {{0}}; + pjmedia_av_sync_setting setting; + + pjmedia_av_sync_setting_default(&setting); + setting.name = (char*) get_fname(filename); + status = pjmedia_av_sync_create(pool, &setting, &avsync); + if (status != PJ_SUCCESS) + goto on_error; + + (*p_streams)->avsync = avsync; + + for (i = 0; i < nstr; i++) { + pjmedia_av_sync_media_setting med_setting; + + pjmedia_av_sync_media_setting_default(&med_setting); + med_setting.type = fport[i]->base.info.fmt.type; + med_setting.name = (char*)pjmedia_type_name(med_setting.type); + if (med_setting.type == PJMEDIA_TYPE_AUDIO) { + med_setting.clock_rate = PJMEDIA_PIA_SRATE(&fport[i]->base.info); + } else if (med_setting.type == PJMEDIA_TYPE_VIDEO) { + med_setting.clock_rate = VIDEO_CLOCK_RATE; + } + status = pjmedia_av_sync_add_media(avsync, &med_setting, + &fport[i]->avsync_media); + if (status != PJ_SUCCESS) + goto on_error; + + /* Set reference timestamps to zeroes */ + status = pjmedia_av_sync_update_ref(fport[i]->avsync_media, + &ts_zero, &ts_zero); + if (status != PJ_SUCCESS) + goto on_error; + + /* Set pointer to AVI streams */ + fport[i]->avi_streams = *p_streams; + } + } + status = pj_grp_lock_add_handler(grp_lock, NULL, *p_streams, &streams_on_destroy); if (status != PJ_SUCCESS) @@ -592,6 +666,15 @@ pjmedia_avi_player_create_streams(pj_pool_t *pool_, for (i = 1; i < nstr; i++) pjmedia_port_destroy(&fport[i]->base); } + + if (*p_streams && (*p_streams)->avsync) { + for (i = 0; i < nstr; i++) { + if (fport[i]->avsync_media) + pjmedia_av_sync_del_media(NULL, fport[i]->avsync_media); + } + pjmedia_av_sync_destroy((*p_streams)->avsync); + } + pj_pool_release(pool); if (status == AVI_EOF) @@ -721,6 +804,108 @@ static pj_status_t file_on_event(pjmedia_event *event, } +static pj_status_t skip_forward(pjmedia_port *this_port, pj_size_t frames) +{ + struct avi_reader_port *fport = (struct avi_reader_port*)this_port; + pj_status_t status = PJ_SUCCESS; + pj_ssize_t remainder = frames; + pjmedia_type type = fport->base.info.fmt.type; + pj_bool_t is_pcm = PJ_FALSE; + + /* For audio, skip current chunk first */ + if (type == PJMEDIA_TYPE_AUDIO) { + is_pcm = (fport->fmt_id!=PJMEDIA_FORMAT_PCMA && + fport->fmt_id!=PJMEDIA_FORMAT_PCMU); + if (fport->size_left > 0) { + pj_ssize_t seek_size = is_pcm? frames * 2 : frames; + seek_size = PJ_MIN(seek_size, fport->size_left); + status = pj_file_setpos(fport->fd, seek_size, PJ_SEEK_CUR); + if (status != PJ_SUCCESS) + return status; + + fport->size_left -= seek_size; + remainder -= (seek_size / (is_pcm? 2 : 1)); + + fport->frame_cnt += (seek_size / (is_pcm? 2 : 1)); + fport->next_ts.u64 = fport->frame_cnt; + } + } + + while (remainder) { + pjmedia_avi_subchunk ch = {0, 0}; + unsigned stream_id; + char *cid; + + /* Need to skip new chunk */ + pj_assert(fport->size_left == 0); + + /* Data is padded to the nearest WORD boundary */ + if (fport->pad) { + status = pj_file_setpos(fport->fd, fport->pad, PJ_SEEK_CUR); + fport->pad = 0; + } + + status = file_read(fport->fd, &ch, sizeof(pjmedia_avi_subchunk)); + if (status != PJ_SUCCESS) + return status; + + PJ_CHECK_OVERFLOW_UINT32_TO_LONG(ch.len, return PJ_EINVAL); + fport->pad = (pj_uint8_t)ch.len & 1; + + cid = (char *)&ch.id; + if (pj_isdigit(cid[0]) && pj_isdigit(cid[1])) + stream_id = (cid[0] - '0') * 10 + (cid[1] - '0'); + else + stream_id = 1000; + + /* We are only interested in data with our stream id */ + if (stream_id != fport->stream_id) { + if (COMPARE_TAG(ch.id, PJMEDIA_AVI_LIST_TAG)) + PJ_LOG(5, (THIS_FILE, "Unsupported LIST tag found in " + "the movi data.")); + else if (COMPARE_TAG(ch.id, PJMEDIA_AVI_RIFF_TAG)) { + PJ_LOG(3, (THIS_FILE, "Unsupported format: multiple " + "AVIs in a single file.")); + return PJ_ENOTSUP; + } + + status = pj_file_setpos(fport->fd, ch.len, PJ_SEEK_CUR); + continue; + } + + /* Found new chunk */ + fport->size_left = ch.len; + if (type == PJMEDIA_TYPE_AUDIO) { + pj_ssize_t seek_size = remainder * (is_pcm? 2 : 1); + seek_size = PJ_MIN(seek_size, fport->size_left); + status = pj_file_setpos(fport->fd, seek_size, PJ_SEEK_CUR); + if (status != PJ_SUCCESS) + return status; + + fport->size_left -= seek_size; + remainder -= (seek_size / (is_pcm? 2 : 1)); + + fport->frame_cnt += (seek_size / (is_pcm? 2 : 1)); + fport->next_ts.u64 = fport->frame_cnt; + } else { + status = pj_file_setpos(fport->fd, fport->size_left, PJ_SEEK_CUR); + if (status != PJ_SUCCESS) + return status; + fport->size_left = 0; + remainder -= 1; + + fport->frame_cnt++; + fport->next_ts.u64 = ((pj_uint64_t)fport->frame_cnt * + VIDEO_CLOCK_RATE * + fport->base.info.fmt.det.vid.fps.denum/ + fport->base.info.fmt.det.vid.fps.num); + } + } /* while (remainder) */ + + return PJ_SUCCESS; +} + + /* * Get frame from file. */ @@ -730,73 +915,162 @@ static pj_status_t avi_get_frame(pjmedia_port *this_port, struct avi_reader_port *fport = (struct avi_reader_port*)this_port; pj_status_t status = PJ_SUCCESS; pj_ssize_t size_read = 0, size_to_read = 0; + pjmedia_port_info* port_info = &fport->base.info; pj_assert(fport->base.info.signature == SIGNATURE); + /* Synchronize media */ + if (fport->avsync_media && !fport->eof) { + pj_int32_t adjust_delay; + + /* Just return if we are increasing delay */ + if (fport->slow_down_frm) { + fport->slow_down_frm--; + frame->type = PJMEDIA_FRAME_TYPE_NONE; + frame->size = 0; + return PJ_SUCCESS; + } + + status = pjmedia_av_sync_update_pts(fport->avsync_media, + &fport->next_ts, &adjust_delay); + if (status == PJ_SUCCESS && adjust_delay) { + pj_ssize_t frames = 0; + + /* If speed up is requested for more than 1 seconds, + * the stream may just be resumed, fast forward. + */ + if (adjust_delay < -1000) { + PJ_LOG(4,(THIS_FILE, "%.*s: %s need to fast forward by %dms", + (int)fport->base.info.name.slen, + fport->base.info.name.ptr, + pjmedia_type_name(port_info->fmt.type), + -adjust_delay)); + + if (fport->base.info.fmt.type == PJMEDIA_TYPE_AUDIO) { + frames = -adjust_delay * + port_info->fmt.det.aud.clock_rate / + 1000; + } else { + frames = -adjust_delay * + port_info->fmt.det.vid.fps.num / + port_info->fmt.det.vid.fps.denum / 1000; + } + status = skip_forward(this_port, frames); + if (status != PJ_SUCCESS) + goto on_error2; + } + + /* Otherwise it is a small adjustment, apply for video stream only + * so the audio playback remains smooth. + */ + else if (port_info->fmt.type == PJMEDIA_TYPE_VIDEO) { + pj_bool_t slowdown = adjust_delay > 0; + + adjust_delay = PJ_ABS(adjust_delay); + frames = adjust_delay * + port_info->fmt.det.vid.fps.num / + port_info->fmt.det.vid.fps.denum / 1000; + + if (slowdown) { + PJ_LOG(4, (THIS_FILE, + "%.*s: video need to slow down by %dms", + (int)port_info->name.slen, + port_info->name.ptr, + adjust_delay)); + frame->type = PJMEDIA_FRAME_TYPE_NONE; + frame->size = 0; + + /* Increase delay */ + fport->slow_down_frm = frames; + return PJ_SUCCESS; + } + + PJ_LOG(4,(THIS_FILE, "%.*s: video need to speed up by %dms", + (int)port_info->name.slen, + port_info->name.ptr, + -adjust_delay)); + status = skip_forward(this_port, (frames? frames : 1)); + if (status != PJ_SUCCESS) + goto on_error2; + } + } + } + + /* Set the frame timestamp */ + frame->timestamp.u64 = fport->next_ts.u64; + /* We encountered end of file */ if (fport->eof) { - PJ_LOG(5,(THIS_FILE, "File port %.*s EOF", - (int)fport->base.info.name.slen, - fport->base.info.name.ptr)); + pj_bool_t no_loop = (fport->options & PJMEDIA_AVI_FILE_NO_LOOP); + pj_bool_t rewind_now = PJ_TRUE; - /* Call callback, if any */ - if (fport->cb2) { - pj_bool_t no_loop = (fport->options & PJMEDIA_AVI_FILE_NO_LOOP); - - if (!fport->subscribed) { - status = pjmedia_event_subscribe(NULL, &file_on_event, - fport, fport); - fport->subscribed = (status == PJ_SUCCESS)? PJ_TRUE: - PJ_FALSE; + /* If synchronized, wait all streams to EOF before rewinding */ + if (fport->avsync_media) { + pjmedia_avi_streams *avi_streams = fport->avi_streams; + + rewind_now = (avi_streams->eof_cnt % avi_streams->num_streams)==0; + if (rewind_now) { + pj_timestamp ts_zero = {{0}}; + pjmedia_av_sync_update_ref(fport->avsync_media, + &ts_zero, &ts_zero); } + } - if (fport->subscribed && fport->eof != 2) { - pjmedia_event event; + /* Call callback, if any */ + if (fport->eof != 2) { - if (no_loop) { - /* To prevent the callback from being called repeatedly */ - fport->eof = 2; - } else { - fport->eof = PJ_FALSE; - pj_file_setpos(fport->fd, fport->start_data, PJ_SEEK_SET); + /* To prevent the callback from being called repeatedly */ + fport->eof = 2; + + if (fport->cb2) { + if (!fport->subscribed) { + status = pjmedia_event_subscribe(NULL, &file_on_event, + fport, fport); + fport->subscribed = (status == PJ_SUCCESS)? PJ_TRUE: + PJ_FALSE; } - pjmedia_event_init(&event, PJMEDIA_EVENT_CALLBACK, - NULL, fport); - pjmedia_event_publish(NULL, fport, &event, - PJMEDIA_EVENT_PUBLISH_POST_EVENT); - } - - /* Should not access player port after this since - * it might have been destroyed by the callback. - */ - frame->type = PJMEDIA_FRAME_TYPE_NONE; - frame->size = 0; - - return (no_loop? PJ_EEOF: PJ_SUCCESS); + if (fport->subscribed) { + pjmedia_event event; - } else if (fport->cb) { - status = (*fport->cb)(this_port, fport->base.port_data.pdata); + pjmedia_event_init(&event, PJMEDIA_EVENT_CALLBACK, + NULL, fport); + pjmedia_event_publish(NULL, fport, &event, + PJMEDIA_EVENT_PUBLISH_POST_EVENT); + } + + /* Should not access player port after this since + * it might have been destroyed by the callback. + */ + frame->type = PJMEDIA_FRAME_TYPE_NONE; + frame->size = 0; + status = PJ_SUCCESS; + + } else if (fport->cb) { + status = (*fport->cb)(this_port, fport->base.port_data.pdata); + } } /* If callback returns non PJ_SUCCESS or 'no loop' is specified, * return immediately (and don't try to access player port since * it might have been destroyed by the callback). */ - if ((status != PJ_SUCCESS) || - (fport->options & PJMEDIA_AVI_FILE_NO_LOOP)) - { + if (status != PJ_SUCCESS || no_loop || !rewind_now) { frame->type = PJMEDIA_FRAME_TYPE_NONE; frame->size = 0; - return PJ_EEOF; + return (no_loop? PJ_EEOF : status); } - /* Rewind file */ - PJ_LOG(5,(THIS_FILE, "File port %.*s rewinding..", - (int)fport->base.info.name.slen, - fport->base.info.name.ptr)); - fport->eof = PJ_FALSE; - pj_file_setpos(fport->fd, fport->start_data, PJ_SEEK_SET); + if (rewind_now) { + PJ_LOG(5,(THIS_FILE, "AVI player port %.*s rewinding..", + (int)fport->base.info.name.slen, + fport->base.info.name.ptr)); + + pj_file_setpos(fport->fd, fport->start_data, PJ_SEEK_SET); + fport->eof = PJ_FALSE; + fport->frame_cnt = 0; + fport->next_ts.u64 = 0; + } } /* For PCMU/A audio stream, reduce frame size to half (temporarily). */ @@ -903,7 +1177,7 @@ static pj_status_t avi_get_frame(pjmedia_port *this_port, break; } while(1); - frame->timestamp.u64 = fport->next_ts.u64; + if (frame->type == PJMEDIA_FRAME_TYPE_AUDIO) { /* Decode PCMU/A frame */ @@ -931,37 +1205,44 @@ static pj_status_t avi_get_frame(pjmedia_port *this_port, frame->size <<= 1; } - if (fport->usec_per_frame) { - fport->next_ts.u64 += (fport->usec_per_frame * - fport->base.info.fmt.det.aud.clock_rate / - 1000000); - } else { - fport->next_ts.u64 += (frame->size * - fport->base.info.fmt.det.aud.clock_rate / - (fport->base.info.fmt.det.aud.avg_bps / 8)); - } + fport->frame_cnt += (frame->size >> 1); + fport->next_ts.u64 = fport->frame_cnt; } else { - if (fport->usec_per_frame) { - fport->next_ts.u64 += (fport->usec_per_frame * VIDEO_CLOCK_RATE / - 1000000); - } else { - fport->next_ts.u64 += (frame->size * VIDEO_CLOCK_RATE / - (fport->base.info.fmt.det.vid.avg_bps / 8)); - } + fport->frame_cnt++; + fport->next_ts.u64 = ((pj_uint64_t)fport->frame_cnt * + VIDEO_CLOCK_RATE * + fport->base.info.fmt.det.vid.fps.denum/ + fport->base.info.fmt.det.vid.fps.num); } return PJ_SUCCESS; on_error2: - if (status == AVI_EOF) { + if (status == AVI_EOF && !fport->eof) { + + /* Reset AV sync on the last stream encountering EOF */ + if (fport->avsync_media) { + pjmedia_avi_streams* avi_streams = fport->avi_streams; + + if (avi_streams->avsync && + (++avi_streams->eof_cnt % avi_streams->num_streams == 0)) + { + pjmedia_av_sync_reset(avi_streams->avsync); + } + } + fport->eof = PJ_TRUE; + PJ_LOG(5,(THIS_FILE, "AVI player port %.*s EOF", + (int)fport->base.info.name.slen, + fport->base.info.name.ptr)); + size_to_read -= size_read; if (size_to_read == (pj_ssize_t)frame->size) { /* Frame is empty */ frame->type = PJMEDIA_FRAME_TYPE_NONE; frame->size = 0; - return PJ_EEOF; + return PJ_EEOF; } pj_bzero((char *)frame->buf + frame->size - size_to_read, size_to_read); @@ -969,7 +1250,9 @@ static pj_status_t avi_get_frame(pjmedia_port *this_port, return PJ_SUCCESS; } - return status; + frame->type = PJMEDIA_FRAME_TYPE_NONE; + frame->size = 0; + return (status==AVI_EOF? PJ_EEOF : status); } /* @@ -988,6 +1271,12 @@ static pj_status_t avi_on_destroy(pjmedia_port *this_port) if (fport->fd != (pj_oshandle_t) (pj_ssize_t)-1) pj_file_close(fport->fd); + + if (fport->avsync_media) { + pjmedia_av_sync_del_media(NULL, fport->avsync_media); + fport->avsync_media = NULL; + } + return PJ_SUCCESS; } diff --git a/pjmedia/src/pjmedia/jbuf.c b/pjmedia/src/pjmedia/jbuf.c index 9fafa6c71b..59070e799a 100644 --- a/pjmedia/src/pjmedia/jbuf.c +++ b/pjmedia/src/pjmedia/jbuf.c @@ -110,6 +110,7 @@ struct pjmedia_jbuf calculation */ int jb_min_shrink_gap; /**< How often can we shrink */ discard_algo jb_discard_algo; /**< Discard algorithm */ + unsigned jb_min_delay; /**< Minimum delay, in frames */ /* Buffer */ jb_framelist_t jb_framelist; /**< the buffer */ @@ -857,9 +858,16 @@ static void jbuf_discard_static(pjmedia_jbuf *jb) * so just disable it when progressive discard is active. */ int diff, burst_level; + unsigned cur_size; burst_level = PJ_MAX(jb->jb_eff_level, jb->jb_level); - diff = jb_framelist_eff_size(&jb->jb_framelist) - burst_level*2; + cur_size = jb_framelist_eff_size(&jb->jb_framelist); + + /* Don't discard if delay is lower than or equal to setting */ + if (cur_size <= jb->jb_min_delay) + return; + + diff = cur_size - burst_level*2; if (diff >= STA_DISC_SAFE_SHRINKING_DIFF) { int seq_origin; @@ -898,10 +906,10 @@ static void jbuf_discard_progressive(pjmedia_jbuf *jb) if (jb->jb_last_op != JB_OP_PUT) return; - /* Check if latency is longer than burst */ + /* Check if latency is longer than burst or minimum delay */ cur_size = jb_framelist_eff_size(&jb->jb_framelist); burst_level = PJ_MAX(jb->jb_eff_level, jb->jb_level); - if (cur_size <= burst_level) { + if (cur_size <= burst_level || cur_size <= jb->jb_min_delay) { /* Reset any scheduled discard */ jb->jb_discard_dist = 0; return; @@ -919,7 +927,7 @@ static void jbuf_discard_progressive(pjmedia_jbuf *jb) (PJMEDIA_JBUF_PRO_DISC_MAX_BURST-PJMEDIA_JBUF_PRO_DISC_MIN_BURST); /* Calculate current discard distance */ - overflow = cur_size - burst_level; + overflow = cur_size - PJ_MAX(burst_level, jb->jb_min_delay); discard_dist = T * jb->jb_frame_ptime_denum / overflow / jb->jb_frame_ptime; @@ -1146,6 +1154,10 @@ PJ_DEF(void) pjmedia_jbuf_get_frame3(pjmedia_jbuf *jb, jb->jb_empty++; + } else if (jb_framelist_eff_size(&jb->jb_framelist) < jb->jb_min_delay) { + *p_frame_type = PJMEDIA_JB_MISSING_FRAME; + if (size) + *size = 0; } else { pjmedia_jb_frame_type ftype = PJMEDIA_JB_NORMAL_FRAME; @@ -1205,6 +1217,7 @@ PJ_DEF(pj_status_t) pjmedia_jbuf_get_state( const pjmedia_jbuf *jb, state->min_prefetch = jb->jb_min_prefetch; state->max_prefetch = jb->jb_max_prefetch; state->max_count = (unsigned)jb->jb_max_count; + state->min_delay_set = jb->jb_min_delay; state->burst = jb->jb_eff_level; state->prefetch = jb->jb_prefetch; @@ -1270,3 +1283,22 @@ PJ_DEF(unsigned) pjmedia_jbuf_remove_frame(pjmedia_jbuf *jb, return count; } + + +PJ_DEF(pj_status_t) pjmedia_jbuf_set_min_delay(pjmedia_jbuf *jb, + unsigned min_delay) +{ + PJ_ASSERT_RETURN(jb, PJ_EINVAL); + + /* Convert milliseconds to frames */ + min_delay *= jb->jb_frame_ptime_denum; + jb->jb_min_delay = min_delay / jb->jb_frame_ptime; + if (min_delay % jb->jb_frame_ptime) + jb->jb_min_delay++; + + /* Should not be higher than half of jitter buffer capacity */ + if (jb->jb_min_delay > jb->jb_max_count/2) + jb->jb_min_delay = (unsigned)jb->jb_max_count/2; + + return PJ_SUCCESS; +} diff --git a/pjmedia/src/pjmedia/rtcp.c b/pjmedia/src/pjmedia/rtcp.c index af080a4e7b..2daa0e3f80 100644 --- a/pjmedia/src/pjmedia/rtcp.c +++ b/pjmedia/src/pjmedia/rtcp.c @@ -574,6 +574,11 @@ static void parse_rtcp_report( pjmedia_rtcp_session *sess, sess->rx_lsr = ((pj_ntohl(sr->ntp_sec) & 0x0000FFFF) << 16) | ((pj_ntohl(sr->ntp_frac) >> 16) & 0xFFFF); + /* Save RTP & NTP timestamps of RTCP packet */ + sess->rx_lsr_ts = pj_ntohl(sr->rtp_ts); + sess->rx_lsr_ntp.u32.hi = pj_ntohl(sr->ntp_sec); + sess->rx_lsr_ntp.u32.lo = pj_ntohl(sr->ntp_frac); + /* Calculate SR arrival time for DLSR */ pj_get_timestamp(&sess->rx_lsr_time); @@ -943,7 +948,7 @@ PJ_DEF(void) pjmedia_rtcp_build_rtcp(pjmedia_rtcp_session *sess, * sent RTCP SR. */ if (sess->stat.tx.pkt != pj_ntohl(sess->rtcp_sr_pkt.sr.sender_pcount)) { - pj_time_val ts_time; + //pj_time_val ts_time; pj_uint32_t rtp_ts; /* So we should send RTCP SR */ @@ -963,11 +968,13 @@ PJ_DEF(void) pjmedia_rtcp_build_rtcp(pjmedia_rtcp_session *sess, sr->ntp_frac = pj_htonl(ntp.lo); /* Fill in RTP timestamp (corresponds to NTP timestamp) in SR. */ - ts_time.sec = ntp.hi - sess->tv_base.sec - JAN_1970; - ts_time.msec = (long)(ntp.lo * 1000.0 / 0xFFFFFFFF); - rtp_ts = sess->rtp_ts_base + - (pj_uint32_t)(sess->clock_rate*ts_time.sec) + - (pj_uint32_t)(sess->clock_rate*ts_time.msec/1000); + // Use real last transmitted RTP timestamp instead of calculated one. + //ts_time.sec = ntp.hi - sess->tv_base.sec - JAN_1970; + //ts_time.msec = (long)(ntp.lo * 1000.0 / 0xFFFFFFFF); + //rtp_ts = sess->rtp_ts_base + + // (pj_uint32_t)(sess->clock_rate*ts_time.sec) + + // (pj_uint32_t)(sess->clock_rate*ts_time.msec/1000); + rtp_ts = sess->stat.rtp_tx_last_ts; sr->rtp_ts = pj_htonl(rtp_ts); TRACE_((sess->name, "TX RTCP SR: ntp_ts=%p", diff --git a/pjmedia/src/pjmedia/stream.c b/pjmedia/src/pjmedia/stream.c index 5bfc261b33..866b9f9eaf 100644 --- a/pjmedia/src/pjmedia/stream.c +++ b/pjmedia/src/pjmedia/stream.c @@ -213,6 +213,7 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) pjmedia_channel *channel = c_strm->dec; unsigned samples_count, samples_per_frame, samples_required; pj_int16_t *p_out_samp; + pj_uint32_t rtp_ts = 0; pj_status_t status; @@ -270,8 +271,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) } /* Get frame from jitter buffer. */ - pjmedia_jbuf_get_frame2(c_strm->jb, channel->buf, &frame_size, - &frame_type, &bit_info); + pjmedia_jbuf_get_frame3(c_strm->jb, channel->buf, &frame_size, + &frame_type, &bit_info, &rtp_ts, NULL); #if TRACE_JB trace_jb_get(c_strm, frame_type, frame_size); @@ -489,6 +490,47 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) } if (!use_dec_buf) samples_count += samples_per_frame; + + /* Update synchronizer with presentation time and check if the + * synchronizer requests for delay adjustment. + */ + if (c_strm->av_sync_media) { + pj_timestamp pts = { 0 }; + pj_int32_t delay_req_ms; + + pts.u32.lo = rtp_ts; + status = pjmedia_av_sync_update_pts(c_strm->av_sync_media, + &pts, &delay_req_ms); + if (status == PJ_SUCCESS && delay_req_ms) { + /* Delay adjustment is requested */ + pjmedia_jb_state jb_state; + int target_delay_ms, cur_delay_ms; + + /* Apply delay request to jitter buffer */ + pjmedia_jbuf_get_state(c_strm->jb, &jb_state); + cur_delay_ms = jb_state.min_delay_set * stream->dec_ptime/ + stream->dec_ptime_denum; + target_delay_ms = cur_delay_ms + delay_req_ms; + if (target_delay_ms < 0) + target_delay_ms = 0; + + /* Just for safety (never see in tests), target delay + * should not exceed 5 seconds. + */ + if (target_delay_ms > 5000) { + PJ_LOG(5,(c_strm->port.info.name.ptr, + "Ignored avsync request for excessive delay" + " (current=%dms, target=%dms)!", + cur_delay_ms, target_delay_ms)); + } else if (cur_delay_ms != target_delay_ms) { + pjmedia_jbuf_set_min_delay(c_strm->jb, + target_delay_ms); + PJ_LOG(5,(c_strm->port.info.name.ptr, + "Adjust audio minimal delay to %dms", + target_delay_ms)); + } + } + } } } @@ -522,6 +564,7 @@ static pj_status_t get_frame_ext( pjmedia_port *port, pjmedia_frame *frame) pjmedia_channel *channel = c_strm->dec; pjmedia_frame_ext *f = (pjmedia_frame_ext*)frame; unsigned samples_per_frame, samples_required; + pj_uint32_t rtp_ts = 0; pj_status_t status; /* Return no frame if channel is paused */ @@ -553,8 +596,8 @@ static pj_status_t get_frame_ext( pjmedia_port *port, pjmedia_frame *frame) pj_mutex_lock( c_strm->jb_mutex ); /* Get frame from jitter buffer. */ - pjmedia_jbuf_get_frame2(c_strm->jb, channel->buf, &frame_size, - &frame_type, &bit_info); + pjmedia_jbuf_get_frame3(c_strm->jb, channel->buf, &frame_size, + &frame_type, &bit_info, &rtp_ts, NULL); #if TRACE_JB trace_jb_get(c_strm, frame_type, frame_size); @@ -595,6 +638,40 @@ static pj_status_t get_frame_ext( pjmedia_port *port, pjmedia_frame *frame) c_strm->jb_last_frm_cnt++; } + /* Update synchronizer with presentation time */ + if (c_strm->av_sync_media) { + pj_timestamp pts = { 0 }; + pj_int32_t delay_req_ms; + + pts.u32.lo = rtp_ts; + status = pjmedia_av_sync_update_pts(c_strm->av_sync_media, + &pts, &delay_req_ms); + if (status == PJ_SUCCESS && delay_req_ms) { + /* Delay adjustment is requested */ + pjmedia_jb_state jb_state; + int target_delay_ms, cur_delay_ms; + + /* Increase delay slowly, but decrease delay quickly */ + if (delay_req_ms > 0) + delay_req_ms = delay_req_ms * 3 / 4; + else + delay_req_ms = delay_req_ms * 4 / 3; + + /* Apply delay request to jitter buffer */ + pjmedia_jbuf_get_state(c_strm->jb, &jb_state); + cur_delay_ms = jb_state.min_delay_set * stream->dec_ptime/ + stream->dec_ptime_denum; + target_delay_ms = cur_delay_ms + delay_req_ms; + if (target_delay_ms < 0) + target_delay_ms = 0; + pjmedia_jbuf_set_min_delay(c_strm->jb, target_delay_ms); + + PJ_LOG(5,(c_strm->port.info.name.ptr, + "Adjust minimal delay to %dms", + target_delay_ms)); + } + } + } else { /* Try to generate frame by invoking PLC (when any) */ @@ -1404,12 +1481,14 @@ static pj_status_t on_stream_rx_rtp(pjmedia_stream_common *c_strm, pj_bool_t *pkt_discarded) { pjmedia_stream *stream = (pjmedia_stream*) c_strm; + pj_timestamp ts; pj_status_t status = PJ_SUCCESS; + /* Get the timestamp from the RTP header */ + ts.u64 = pj_ntohl(hdr->ts); + /* Handle incoming DTMF. */ if (hdr->pt == stream->rx_event_pt) { - pj_timestamp ts; - /* Ignore out-of-order packet as it will be detected as new * digit. Also ignore duplicate packet as it serves no use. */ @@ -1417,9 +1496,6 @@ static pj_status_t on_stream_rx_rtp(pjmedia_stream_common *c_strm, goto on_return; } - /* Get the timestamp of the event */ - ts.u64 = pj_ntohl(hdr->ts); - handle_incoming_dtmf(stream, &ts, payload, payloadlen); goto on_return; } @@ -1438,15 +1514,11 @@ static pj_status_t on_stream_rx_rtp(pjmedia_stream_common *c_strm, * to ask the codec to "parse" the payload into multiple frames. */ enum { MAX = 16 }; - pj_timestamp ts; unsigned i, count = MAX; unsigned ts_span; pjmedia_frame frames[MAX]; pj_bzero(frames, sizeof(frames[0]) * MAX); - /* Get the timestamp of the first sample */ - ts.u64 = pj_ntohl(hdr->ts); - /* Parse the payload. */ status = pjmedia_codec_parse(stream->codec, (void*)payload, payloadlen, &ts, &count, frames); @@ -1587,8 +1659,9 @@ static pj_status_t on_stream_rx_rtp(pjmedia_stream_common *c_strm, pj_bool_t discarded; ext_seq = (unsigned)(frames[i].timestamp.u64 / ts_span); - pjmedia_jbuf_put_frame2(c_strm->jb, frames[i].buf, frames[i].size, - frames[i].bit_info, ext_seq, &discarded); + pjmedia_jbuf_put_frame3(c_strm->jb, frames[i].buf, frames[i].size, + frames[i].bit_info, ext_seq, ts.u32.lo, + &discarded); if (discarded) *pkt_discarded = PJ_TRUE; } diff --git a/pjmedia/src/pjmedia/stream_common.c b/pjmedia/src/pjmedia/stream_common.c index 6127a02f6a..72e98b97da 100644 --- a/pjmedia/src/pjmedia/stream_common.c +++ b/pjmedia/src/pjmedia/stream_common.c @@ -488,6 +488,49 @@ pjmedia_stream_common_get_rtp_session_info(pjmedia_stream_common *c_strm, return PJ_SUCCESS; } + +/* + * Set media presentation synchronizer. + */ +PJ_DEF(pj_status_t) +pjmedia_stream_common_set_avsync(pjmedia_stream_common* stream, + pjmedia_av_sync* av_sync) +{ + pj_status_t status = PJ_SUCCESS; + + PJ_ASSERT_RETURN(stream, PJ_EINVAL); + + /* First, remove existing */ + if (stream->av_sync && stream->av_sync_media) { + status = pjmedia_av_sync_del_media(stream->av_sync, + stream->av_sync_media); + stream->av_sync = NULL; + stream->av_sync_media = NULL; + } + + /* Then set a new or reset */ + if (av_sync) { + pjmedia_av_sync_media_setting setting; + + pjmedia_av_sync_media_setting_default(&setting); + setting.type = stream->si->type; + if (stream->si->type == PJMEDIA_TYPE_AUDIO) { + setting.name = "Audio"; + setting.clock_rate = PJMEDIA_PIA_SRATE(&stream->port.info); + } else if (stream->si->type == PJMEDIA_TYPE_VIDEO) { + setting.name = "Video"; + setting.clock_rate = 90000; + } + + stream->av_sync = av_sync; + status = pjmedia_av_sync_add_media(av_sync, &setting, + &stream->av_sync_media); + } + + return status; +} + + static pj_status_t build_rtcp_fb(pjmedia_stream_common *c_strm, void *buf, pj_size_t *length) { diff --git a/pjmedia/src/pjmedia/stream_imp_common.c b/pjmedia/src/pjmedia/stream_imp_common.c index aa35c6983b..5bae4b1f09 100755 --- a/pjmedia/src/pjmedia/stream_imp_common.c +++ b/pjmedia/src/pjmedia/stream_imp_common.c @@ -320,6 +320,14 @@ static void on_rx_rtcp( void *data, } pjmedia_rtcp_rx_rtcp(&c_strm->rtcp, pkt, bytes_read); + + /* Update synchronizer with reference time from RTCP-SR */ + if (c_strm->av_sync_media && c_strm->rtcp.rx_lsr_ts) { + pj_timestamp ntp = {0}, ts = {0}; + ntp = c_strm->rtcp.rx_lsr_ntp; + ts.u32.lo = c_strm->rtcp.rx_lsr_ts; + pjmedia_av_sync_update_ref(c_strm->av_sync_media, &ntp, &ts); + } } /* @@ -588,6 +596,10 @@ static void on_destroy(void *arg) c_strm->jb = NULL; } + /* Destroy media synchronizer */ + if (c_strm->av_sync && c_strm->av_sync_media) + pjmedia_stream_common_set_avsync(c_strm, NULL); + #if TRACE_JB if (TRACE_JB_OPENED(c_strm)) { pj_file_close(c_strm->trace_jb_fd); diff --git a/pjmedia/src/pjmedia/vid_stream.c b/pjmedia/src/pjmedia/vid_stream.c index f8479f5c59..c436880da4 100644 --- a/pjmedia/src/pjmedia/vid_stream.c +++ b/pjmedia/src/pjmedia/vid_stream.c @@ -101,6 +101,9 @@ struct pjmedia_vid_stream pjmedia_ratio dec_max_fps; /**< Max fps of decoding dir. */ pjmedia_frame dec_frame; /**< Current decoded frame. */ unsigned dec_delay_cnt; /**< Decoding delay (in frames).*/ + unsigned dec_add_delay_cnt; + /**< Decoding additional delay + for sync (in frames). */ unsigned dec_max_delay; /**< Decoding max delay (in ts).*/ pjmedia_event fmt_event; /**< Buffered fmt_changed event to avoid deadlock */ @@ -353,6 +356,7 @@ static pj_status_t detach_send_manager(send_stream *ss) } e = next; } + ss->mgr = NULL; pj_grp_lock_release(mgr->grp_lock); /* Decrease ref counter */ @@ -373,6 +377,13 @@ static send_entry* get_send_entry(send_stream *ss) pj_sub_timestamp(&min_sent_ts, &idle_ts); pj_grp_lock_acquire(ss->grp_lock); + + /* Make sure send manager has not been detached */ + if (ss->mgr == NULL) { + pj_grp_lock_release(ss->grp_lock); + return NULL; + } + e = ss->free_list.next; while (e != &ss->free_list) { send_entry *next = e->next; @@ -409,6 +420,12 @@ static void send_rtp(send_stream *ss, send_entry *entry) pj_grp_lock_acquire(ss->grp_lock); + /* Make sure send manager has not been detached */ + if (ss->mgr == NULL) { + pj_grp_lock_release(ss->grp_lock); + return; + } + /* Calculate earliest sending time allowed by rate control */ ss->rc_total += entry->buf_size; send_ts.u64 = ss->rc_total * ss->ts_freq.u64 * 8 / ss->rc_bandwidth; @@ -646,9 +663,11 @@ static pj_status_t on_stream_rx_rtp(pjmedia_stream_common *c_strm, } if (can_decode) { + pj_size_t old_size = stream->dec_frame.size; + stream->dec_frame.size = stream->dec_max_size; if (decode_frame(stream, &stream->dec_frame) != PJ_SUCCESS) { - stream->dec_frame.size = 0; + stream->dec_frame.size = old_size; } } } @@ -1049,7 +1068,9 @@ static pj_status_t decode_frame(pjmedia_vid_stream *stream, frm_pkt_cnt = cnt; /* Is it time to decode? Check with minimum delay setting */ - if (++frm_cnt == stream->dec_delay_cnt) { + if (++frm_cnt == stream->dec_delay_cnt + + stream->dec_add_delay_cnt) + { got_frame = PJ_TRUE; break; } @@ -1299,6 +1320,95 @@ static pj_status_t get_frame(pjmedia_port *port, pj_grp_lock_release( c_strm->grp_lock ); + /* Update synchronizer with presentation time */ + if (c_strm->av_sync_media && frame->type != PJMEDIA_FRAME_TYPE_NONE) { + pj_timestamp pts = { 0 }; + pj_int32_t delay_req_ms; + pj_status_t status; + + pts = frame->timestamp; + status = pjmedia_av_sync_update_pts(c_strm->av_sync_media, &pts, + &delay_req_ms); + if (status == PJ_SUCCESS && delay_req_ms) { + /* Delay adjustment is requested */ + int target_delay_ms, cur_delay_ms, target_delay_cnt; + unsigned last_add_delay_cnt; + + /* Apply delay request */ + last_add_delay_cnt = stream->dec_add_delay_cnt; + cur_delay_ms = (stream->dec_delay_cnt+stream->dec_add_delay_cnt) * + 1000 * + stream->dec_max_fps.denum / + stream->dec_max_fps.num; + target_delay_ms = cur_delay_ms + delay_req_ms; + target_delay_cnt = target_delay_ms * stream->dec_max_fps.num / + stream->dec_max_fps.denum / 1000; + if (target_delay_cnt <= (int)stream->dec_delay_cnt) + stream->dec_add_delay_cnt = 0; + else + stream->dec_add_delay_cnt = target_delay_cnt - + stream->dec_delay_cnt; + + /* Just for safety (never see in tests), target delay should not + * exceed 5 seconds. + */ + if (target_delay_ms > 5000) { + stream->dec_add_delay_cnt = last_add_delay_cnt; + PJ_LOG(5,(c_strm->port.info.name.ptr, + "Ignored avsync request for excessive delay" + " (current=%dms, target=%dms)!", + cur_delay_ms, target_delay_ms)); + } + + if (stream->dec_add_delay_cnt != last_add_delay_cnt) { + PJ_LOG(5,(c_strm->port.info.name.ptr, + "Adjust video minimal delay to (%d+%d) frames", + stream->dec_delay_cnt, stream->dec_add_delay_cnt)); + } + + /* When requested to speed-up, try to skip frames */ + if (delay_req_ms < 0) { + enum { MAX_SKIP_MS = 500 }; + unsigned cnt = 0, max_cnt; + + /* Translate milliseconds to number of frames to drop, + * apply upper limit to avoid blocking too long. + */ + max_cnt = PJ_MIN(-delay_req_ms, MAX_SKIP_MS) * + stream->dec_max_fps.num / + stream->dec_max_fps.denum / 1000; + + /* Round up one frame, the decoded frame will be played later + * in the next get_frame(). + */ + max_cnt++; + + while (cnt < max_cnt) { + pj_size_t old_size; + + pj_grp_lock_acquire( c_strm->grp_lock ); + old_size = stream->dec_frame.size; + stream->dec_frame.size = stream->dec_max_size; + if (decode_frame(stream, &stream->dec_frame)==PJ_SUCCESS) + { + cnt++; + } else { + /* Revert dec_frame to last successful decoding */ + stream->dec_frame.size = old_size; + pj_grp_lock_release( c_strm->grp_lock ); + break; + } + pj_grp_lock_release( c_strm->grp_lock ); + } + + if (cnt) { + PJ_LOG(5,(c_strm->port.info.name.ptr, + "Skipped %d frames to reduce delay", cnt)); + } + } + } + } + return PJ_SUCCESS; } @@ -1890,8 +2000,11 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_destroy( pjmedia_vid_stream *stream ) c_strm->dec->port.get_frame = NULL; /* Detach from sending manager */ - if (stream->send_stream) + if (stream->send_stream) { + pj_grp_lock_acquire(c_strm->grp_lock); detach_send_manager(stream->send_stream); + pj_grp_lock_release(c_strm->grp_lock); + } #if TRACE_RC { diff --git a/pjsip-apps/src/swig/symbols.i b/pjsip-apps/src/swig/symbols.i index 8f26b22be6..ac7456db3a 100644 --- a/pjsip-apps/src/swig/symbols.i +++ b/pjsip-apps/src/swig/symbols.i @@ -451,7 +451,8 @@ typedef enum pjmedia_vid_packing typedef enum pjmedia_vid_stream_rc_method { PJMEDIA_VID_STREAM_RC_NONE = 0, - PJMEDIA_VID_STREAM_RC_SIMPLE_BLOCKING = 1 + PJMEDIA_VID_STREAM_RC_SIMPLE_BLOCKING = 1, + PJMEDIA_VID_STREAM_RC_SEND_THREAD = 2 } pjmedia_vid_stream_rc_method; enum pjmedia_file_writer_option @@ -932,7 +933,8 @@ typedef enum pjsua_call_flag PJSUA_CALL_REINIT_MEDIA = 16, PJSUA_CALL_UPDATE_VIA = 32, PJSUA_CALL_UPDATE_TARGET = 64, - PJSUA_CALL_SET_MEDIA_DIR = 128 + PJSUA_CALL_SET_MEDIA_DIR = 128, + PJSUA_CALL_NO_MEDIA_SYNC = 256 } pjsua_call_flag; typedef enum pjsua_create_media_transport_flag diff --git a/pjsip/include/pjsua-lib/pjsua.h b/pjsip/include/pjsua-lib/pjsua.h index 3ee85bec84..a41209d630 100644 --- a/pjsip/include/pjsua-lib/pjsua.h +++ b/pjsip/include/pjsua-lib/pjsua.h @@ -5578,7 +5578,12 @@ typedef enum pjsua_call_flag /** * Set media direction as specified in pjsua_call_setting.media_dir. */ - PJSUA_CALL_SET_MEDIA_DIR = 128 + PJSUA_CALL_SET_MEDIA_DIR = 128, + + /** + * Disable inter-media synchronization. + */ + PJSUA_CALL_NO_MEDIA_SYNC = 256 } pjsua_call_flag; diff --git a/pjsip/include/pjsua-lib/pjsua_internal.h b/pjsip/include/pjsua-lib/pjsua_internal.h index b14f658730..bb435b8f45 100644 --- a/pjsip/include/pjsua-lib/pjsua_internal.h +++ b/pjsip/include/pjsua-lib/pjsua_internal.h @@ -242,6 +242,8 @@ struct pjsua_call pj_str_t hangup_reason; /**< Hangup reason. */ pjsua_msg_data *hangup_msg_data;/**< Hangup message data. */ pj_str_t siprec_metadata;/** siprec metadata in body */ + + pjmedia_av_sync *av_sync; /**< Media stream synchronizer */ }; diff --git a/pjsip/src/pjsua-lib/pjsua_aud.c b/pjsip/src/pjsua-lib/pjsua_aud.c index 689e0cf6fd..0149c95316 100644 --- a/pjsip/src/pjsua-lib/pjsua_aud.c +++ b/pjsip/src/pjsua-lib/pjsua_aud.c @@ -781,6 +781,15 @@ pj_status_t pjsua_aud_channel_update(pjsua_call_media *call_med, goto on_return; } + /* Add stream to synchronizer */ + if (call->av_sync) { + status = pjmedia_stream_common_set_avsync( + (pjmedia_stream_common*)call_med->strm.a.stream, + call->av_sync); + if (status != PJ_SUCCESS) + goto on_return; + } + /* Start stream */ status = pjmedia_stream_start(call_med->strm.a.stream); if (status != PJ_SUCCESS) { diff --git a/pjsip/src/pjsua-lib/pjsua_media.c b/pjsip/src/pjsua-lib/pjsua_media.c index 43e3960ecf..31938bd484 100644 --- a/pjsip/src/pjsua-lib/pjsua_media.c +++ b/pjsip/src/pjsua-lib/pjsua_media.c @@ -3428,8 +3428,14 @@ pj_status_t pjsua_media_channel_deinit(pjsua_call_id call_id) if (dlg && pj_log_get_level() >= 3) log_call_dump(call_id); + /* Stop all media */ stop_media_session(call_id); + /* Destroy media synchronizer */ + if (call->av_sync) + pjmedia_av_sync_destroy(call->av_sync); + call->av_sync = NULL; + /* Stop trickle ICE timer */ if (call->trickle_ice.trickling > PJSUA_OP_STATE_NULL) { call->trickle_ice.trickling = PJSUA_OP_STATE_NULL; @@ -4386,6 +4392,34 @@ pj_status_t pjsua_media_channel_update(pjsua_call_id call_id, pj_memcpy(call->media, call->media_prov, sizeof(call->media_prov[0]) * call->med_prov_cnt); + /* Create/reset synchronizer */ + if ((call->opt.flag & PJSUA_CALL_NO_MEDIA_SYNC)==0 && + (maudcnt+mvidcnt) > 1) + { + if (call->av_sync) { + pjmedia_av_sync_reset(call->av_sync); + } else { + pjmedia_av_sync_setting setting; + char name[PJ_MAX_OBJ_NAME]; + + pj_ansi_snprintf(name, sizeof(name), "avsync-call_%02d", call_id); + pjmedia_av_sync_setting_default(&setting); + setting.is_streaming = PJ_TRUE; + setting.name = name; + status = pjmedia_av_sync_create(tmp_pool, &setting, &call->av_sync); + if (status != PJ_SUCCESS) { + PJ_PERROR(3, (THIS_FILE, status, + "Call %d: Failed to create synchronizer", call_id)); + } + } + } + + /* Destroy existing synchronizer if synchronization is cancelled */ + else if ((call->opt.flag & PJSUA_CALL_NO_MEDIA_SYNC) && call->av_sync) { + pjmedia_av_sync_destroy(call->av_sync); + call->av_sync = NULL; + } + /* Process each media stream */ for (mi=0; mi < call->med_cnt; ++mi) { pjsua_call_media *call_med = &call->media[mi]; diff --git a/pjsip/src/pjsua-lib/pjsua_vid.c b/pjsip/src/pjsua-lib/pjsua_vid.c index 6204a29b53..24fb0d3e54 100644 --- a/pjsip/src/pjsua-lib/pjsua_vid.c +++ b/pjsip/src/pjsua-lib/pjsua_vid.c @@ -1196,6 +1196,15 @@ pj_status_t pjsua_vid_channel_update(pjsua_call_media *call_med, if (status != PJ_SUCCESS) goto on_error; + /* Add stream to synchronizer */ + if (call->av_sync) { + status = pjmedia_stream_common_set_avsync( + (pjmedia_stream_common*)call_med->strm.v.stream, + call->av_sync); + if (status != PJ_SUCCESS) + goto on_error; + } + /* Subscribe to video stream events */ pjmedia_event_subscribe(NULL, &call_media_on_event, call_med, call_med->strm.v.stream);