From 26c2f7f11a4b55d7b375f99cd3e0ca9c95e0b762 Mon Sep 17 00:00:00 2001 From: sauwming Date: Wed, 29 Jun 2022 13:06:31 +0800 Subject: [PATCH 1/2] Perform async rendering in the main thread on Mac & iOS --- pjmedia/src/pjmedia-videodev/darwin_dev.m | 83 +++++++++++++------ pjmedia/src/pjmedia-videodev/ios_opengl_dev.m | 60 +++++++++++++- .../xcshareddata/xcschemes/ipjsua.xcscheme | 1 + 3 files changed, 116 insertions(+), 28 deletions(-) diff --git a/pjmedia/src/pjmedia-videodev/darwin_dev.m b/pjmedia/src/pjmedia-videodev/darwin_dev.m index 5de6e475bf..48935a588f 100644 --- a/pjmedia/src/pjmedia-videodev/darwin_dev.m +++ b/pjmedia/src/pjmedia-videodev/darwin_dev.m @@ -148,6 +148,9 @@ @interface VOutDelegate: NSObject AVCaptureVideoPreviewLayer *prev_layer; #if TARGET_OS_IPHONE + pj_bool_t is_running; + pj_bool_t is_rendering; + NSLock *render_lock; void *render_buf; pj_size_t render_buf_size; CGDataProviderRef render_data_provider; @@ -512,9 +515,11 @@ static pj_status_t darwin_factory_default_param(pj_pool_t *pool, @implementation VOutDelegate #if TARGET_OS_IPHONE - (void)update_image -{ +{ NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + [stream->render_lock lock]; + CGImageRef cgIm = CGImageCreate(stream->size.w, stream->size.h, 8, 32, stream->bytes_per_row, colorSpace, kCGImageAlphaFirst | @@ -524,10 +529,20 @@ - (void)update_image CGColorSpaceRelease(colorSpace); stream->render_view.layer.contents = (__bridge id)(cgIm); + [stream->render_lock unlock]; CGImageRelease(cgIm); [pool release]; + stream->is_rendering = PJ_FALSE; } + +- (void)finish_render +{ + /* Do nothing. This function is serialized in the main thread, so when + * it is called, we can be sure that update_image() has completed. + */ +} + #endif - (void)session_runtime_error:(NSNotification *)notification @@ -962,6 +977,7 @@ static pj_status_t darwin_factory_create_stream( strm->vout_delegate->stream = strm; } + strm->render_lock = [NSLock alloc]; strm->render_buf = pj_pool_alloc(pool, strm->frame_size); strm->render_buf_size = strm->frame_size; strm->render_data_provider = CGDataProviderCreateWithData(NULL, @@ -1311,14 +1327,14 @@ static pj_status_t darwin_stream_start(pjmedia_vid_dev_stream *strm) { struct darwin_stream *stream = (struct darwin_stream*)strm; - PJ_UNUSED_ARG(stream); - PJ_LOG(4, (THIS_FILE, "Starting Darwin video stream")); + stream->is_running = PJ_TRUE; + if (stream->cap_session) { - dispatch_sync_on_main_queue(^{ - [stream->cap_session startRunning]; - }); + [stream->cap_session + performSelectorOnMainThread:@selector(startRunning) + withObject:nil waitUntilDone:YES]; if (![stream->cap_session isRunning]) { /* More info about the error should be reported in @@ -1346,16 +1362,25 @@ static pj_status_t darwin_stream_put_frame(pjmedia_vid_dev_stream *strm, */ if (frame->size==0 || frame->buf==NULL) return PJ_SUCCESS; - + + if (!stream->is_running) + return PJ_EINVALIDOP; + + /* Prevent more than one async rendering task. */ + if (stream->is_rendering) + return PJ_EIGNORED; + + [stream->render_lock lock]; if (stream->frame_size >= frame->size) pj_memcpy(stream->render_buf, frame->buf, frame->size); else pj_memcpy(stream->render_buf, frame->buf, stream->frame_size); + [stream->render_lock lock]; - /* Perform video display in a background thread */ - dispatch_sync_on_main_queue(^{ - [stream->vout_delegate update_image]; - }); + /* Perform video display in the main thread */ + stream->is_rendering = PJ_TRUE; + [stream->vout_delegate performSelectorOnMainThread:@selector(update_image) + withObject:nil waitUntilDone:NO]; #endif return PJ_SUCCESS; @@ -1371,10 +1396,16 @@ static pj_status_t darwin_stream_stop(pjmedia_vid_dev_stream *strm) PJ_LOG(4, (THIS_FILE, "Stopping Darwin video stream")); - dispatch_sync_on_main_queue(^{ - [stream->cap_session stopRunning]; - }); + [stream->cap_session performSelectorOnMainThread:@selector(stopRunning) + withObject:nil waitUntilDone:YES]; stream->has_image = PJ_FALSE; + stream->is_running = PJ_FALSE; + +#if TARGET_OS_IPHONE + /* Wait until the rendering finishes */ + [stream->vout_delegate performSelectorOnMainThread:@selector(finish_render) + withObject:nil waitUntilDone:YES]; +#endif return PJ_SUCCESS; } @@ -1411,20 +1442,19 @@ static pj_status_t darwin_stream_destroy(pjmedia_vid_dev_stream *strm) #if TARGET_OS_IPHONE if (stream->prev_layer) { - CALayer *prev_layer = stream->prev_layer; - dispatch_sync_on_main_queue(^{ - [prev_layer removeFromSuperlayer]; - [prev_layer release]; - }); + [stream->prev_layer + performSelectorOnMainThread:@selector(removeFromSuperlayer) + withObject:nil waitUntilDone:YES]; + [stream->prev_layer release]; stream->prev_layer = nil; } if (stream->render_view) { - UIView *view = stream->render_view; - dispatch_sync_on_main_queue(^{ - [view removeFromSuperview]; - [view release]; - }); + [stream->render_view + performSelectorOnMainThread:@selector(removeFromSuperview) + withObject:nil waitUntilDone:YES]; + + [stream->render_view release]; stream->render_view = nil; } @@ -1432,6 +1462,11 @@ static pj_status_t darwin_stream_destroy(pjmedia_vid_dev_stream *strm) CGDataProviderRelease(stream->render_data_provider); stream->render_data_provider = nil; } + + if (stream->render_lock) { + [stream->render_lock release]; + stream->render_lock = nil; + } #endif /* TARGET_OS_IPHONE */ if (stream->queue) { diff --git a/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m b/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m index e04612f765..ef671ef4cf 100644 --- a/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m +++ b/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m @@ -72,7 +72,12 @@ @interface GLView : UIView pj_timestamp frame_ts; unsigned ts_inc; pjmedia_rect_size vid_size; - const pjmedia_frame *frame; + unsigned frame_size; + + pj_bool_t is_rendering; + NSLock *buf_lock; + void *render_buf; + unsigned render_buf_size; gl_buffers *gl_buf; GLView *gl_view; @@ -228,10 +233,14 @@ - (void)render return; } + [stream->buf_lock lock]; pjmedia_vid_dev_opengl_draw(stream->gl_buf, stream->vid_size.w, stream->vid_size.h, - stream->frame->buf); + stream->render_buf); + [stream->buf_lock unlock]; [stream->ogl_context presentRenderbuffer:GL_RENDERBUFFER]; + + stream->is_rendering = PJ_FALSE; } - (void)finish_render @@ -262,7 +271,9 @@ - (void)change_format pjmedia_vid_dev_stream **p_vid_strm) { struct iosgl_stream *strm; + const pjmedia_video_format_info *vfi; const pjmedia_video_format_detail *vfd; + pjmedia_video_apply_fmt_param vafp; pj_status_t status = PJ_SUCCESS; CGRect rect; @@ -329,6 +340,18 @@ - (void)change_format iosgl_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_ORIENTATION, ¶m->orient); } + + vfi = pjmedia_get_video_format_info(NULL, param->fmt.id); + if (!vfi) return PJMEDIA_EVID_BADFORMAT; + vafp.size = param->fmt.det.vid.size; + vafp.buffer = NULL; + if (vfi->apply_fmt(vfi, &vafp) != PJ_SUCCESS) + return PJMEDIA_EVID_BADFORMAT; + + strm->frame_size = vafp.framebytes; + strm->render_buf_size = strm->frame_size; + strm->render_buf = pj_pool_alloc(strm->pool, strm->render_buf_size); + strm->buf_lock = [NSLock alloc]; PJ_LOG(4, (THIS_FILE, "iOS OpenGL ES renderer successfully created")); @@ -396,6 +419,7 @@ static pj_status_t iosgl_stream_set_cap(pjmedia_vid_dev_stream *s, if (cap==PJMEDIA_VID_DEV_CAP_FORMAT) { const pjmedia_video_format_info *vfi; + pjmedia_video_apply_fmt_param vafp; pjmedia_format *fmt = (pjmedia_format *)pval; iosgl_fmt_info *ifi; @@ -406,8 +430,20 @@ static pj_status_t iosgl_stream_set_cap(pjmedia_vid_dev_stream *s, fmt->id); if (!vfi) return PJMEDIA_EVID_BADFORMAT; + + vafp.size = fmt->det.vid.size; + vafp.buffer = NULL; + if (vfi->apply_fmt(vfi, &vafp) != PJ_SUCCESS) + return PJMEDIA_EVID_BADFORMAT; pjmedia_format_copy(&strm->param.fmt, fmt); + + strm->frame_size = vafp.framebytes; + if (strm->render_buf_size < strm->frame_size) { + /* Realloc only when needed */ + strm->render_buf_size = strm->frame_size; + strm->render_buf=pj_pool_alloc(strm->pool, strm->render_buf_size); + } [strm->gl_view performSelectorOnMainThread:@selector(change_format) withObject:nil waitUntilDone:YES]; @@ -480,10 +516,21 @@ static pj_status_t iosgl_stream_put_frame(pjmedia_vid_dev_stream *strm, if (!stream->is_running) return PJ_EINVALIDOP; - stream->frame = frame; + /* Prevent more than one async rendering task. */ + if (stream->is_rendering) + return PJ_EIGNORED; + + [stream->buf_lock lock]; + if (stream->frame_size >= frame->size) + pj_memcpy(stream->render_buf, frame->buf, frame->size); + else + pj_memcpy(stream->render_buf, frame->buf, stream->frame_size); + [stream->buf_lock unlock]; + /* Perform OpenGL drawing in the main thread. */ + stream->is_rendering = PJ_TRUE; [stream->gl_view performSelectorOnMainThread:@selector(render) - withObject:nil waitUntilDone:YES]; + withObject:nil waitUntilDone:NO]; return PJ_SUCCESS; } @@ -513,6 +560,11 @@ static pj_status_t iosgl_stream_destroy(pjmedia_vid_dev_stream *strm) if (stream->is_running) iosgl_stream_stop(strm); + + if (stream->buf_lock) { + [stream->buf_lock release]; + stream->buf_lock = NULL; + } if (stream->gl_view) { [stream->gl_view performSelectorOnMainThread:@selector(deinit_gl) diff --git a/pjsip-apps/src/pjsua/ios/ipjsua.xcodeproj/xcshareddata/xcschemes/ipjsua.xcscheme b/pjsip-apps/src/pjsua/ios/ipjsua.xcodeproj/xcshareddata/xcschemes/ipjsua.xcscheme index 36858e2d42..e98ec924ef 100644 --- a/pjsip-apps/src/pjsua/ios/ipjsua.xcodeproj/xcshareddata/xcschemes/ipjsua.xcscheme +++ b/pjsip-apps/src/pjsua/ios/ipjsua.xcodeproj/xcshareddata/xcschemes/ipjsua.xcscheme @@ -48,6 +48,7 @@ ignoresPersistentStateOnLaunch = "NO" debugDocumentVersioning = "YES" debugServiceExtension = "internal" + enableGPUValidationMode = "1" allowLocationSimulation = "YES"> From 9c400f1048c4af107a42ae1064a8e2155f0fe537 Mon Sep 17 00:00:00 2001 From: sauwming Date: Wed, 29 Jun 2022 13:39:27 +0800 Subject: [PATCH 2/2] Remove unnecessary(?) locking --- pjmedia/src/pjmedia-videodev/darwin_dev.m | 13 +------------ pjmedia/src/pjmedia-videodev/ios_opengl_dev.m | 11 ----------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/pjmedia/src/pjmedia-videodev/darwin_dev.m b/pjmedia/src/pjmedia-videodev/darwin_dev.m index 48935a588f..1cbe45ad41 100644 --- a/pjmedia/src/pjmedia-videodev/darwin_dev.m +++ b/pjmedia/src/pjmedia-videodev/darwin_dev.m @@ -150,7 +150,6 @@ @interface VOutDelegate: NSObject #if TARGET_OS_IPHONE pj_bool_t is_running; pj_bool_t is_rendering; - NSLock *render_lock; void *render_buf; pj_size_t render_buf_size; CGDataProviderRef render_data_provider; @@ -518,7 +517,6 @@ - (void)update_image { NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); - [stream->render_lock lock]; CGImageRef cgIm = CGImageCreate(stream->size.w, stream->size.h, 8, 32, stream->bytes_per_row, colorSpace, @@ -529,7 +527,6 @@ - (void)update_image CGColorSpaceRelease(colorSpace); stream->render_view.layer.contents = (__bridge id)(cgIm); - [stream->render_lock unlock]; CGImageRelease(cgIm); [pool release]; @@ -977,7 +974,6 @@ static pj_status_t darwin_factory_create_stream( strm->vout_delegate->stream = strm; } - strm->render_lock = [NSLock alloc]; strm->render_buf = pj_pool_alloc(pool, strm->frame_size); strm->render_buf_size = strm->frame_size; strm->render_data_provider = CGDataProviderCreateWithData(NULL, @@ -1370,12 +1366,10 @@ static pj_status_t darwin_stream_put_frame(pjmedia_vid_dev_stream *strm, if (stream->is_rendering) return PJ_EIGNORED; - [stream->render_lock lock]; if (stream->frame_size >= frame->size) pj_memcpy(stream->render_buf, frame->buf, frame->size); else pj_memcpy(stream->render_buf, frame->buf, stream->frame_size); - [stream->render_lock lock]; /* Perform video display in the main thread */ stream->is_rendering = PJ_TRUE; @@ -1461,12 +1455,7 @@ static pj_status_t darwin_stream_destroy(pjmedia_vid_dev_stream *strm) if (stream->render_data_provider) { CGDataProviderRelease(stream->render_data_provider); stream->render_data_provider = nil; - } - - if (stream->render_lock) { - [stream->render_lock release]; - stream->render_lock = nil; - } + } #endif /* TARGET_OS_IPHONE */ if (stream->queue) { diff --git a/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m b/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m index ef671ef4cf..51f6e67ce5 100644 --- a/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m +++ b/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m @@ -75,7 +75,6 @@ @interface GLView : UIView unsigned frame_size; pj_bool_t is_rendering; - NSLock *buf_lock; void *render_buf; unsigned render_buf_size; @@ -233,10 +232,8 @@ - (void)render return; } - [stream->buf_lock lock]; pjmedia_vid_dev_opengl_draw(stream->gl_buf, stream->vid_size.w, stream->vid_size.h, stream->render_buf); - [stream->buf_lock unlock]; [stream->ogl_context presentRenderbuffer:GL_RENDERBUFFER]; @@ -351,7 +348,6 @@ - (void)change_format strm->frame_size = vafp.framebytes; strm->render_buf_size = strm->frame_size; strm->render_buf = pj_pool_alloc(strm->pool, strm->render_buf_size); - strm->buf_lock = [NSLock alloc]; PJ_LOG(4, (THIS_FILE, "iOS OpenGL ES renderer successfully created")); @@ -520,12 +516,10 @@ static pj_status_t iosgl_stream_put_frame(pjmedia_vid_dev_stream *strm, if (stream->is_rendering) return PJ_EIGNORED; - [stream->buf_lock lock]; if (stream->frame_size >= frame->size) pj_memcpy(stream->render_buf, frame->buf, frame->size); else pj_memcpy(stream->render_buf, frame->buf, stream->frame_size); - [stream->buf_lock unlock]; /* Perform OpenGL drawing in the main thread. */ stream->is_rendering = PJ_TRUE; @@ -561,11 +555,6 @@ static pj_status_t iosgl_stream_destroy(pjmedia_vid_dev_stream *strm) if (stream->is_running) iosgl_stream_stop(strm); - if (stream->buf_lock) { - [stream->buf_lock release]; - stream->buf_lock = NULL; - } - if (stream->gl_view) { [stream->gl_view performSelectorOnMainThread:@selector(deinit_gl) withObject:nil waitUntilDone:YES];