Skip to content

Perform async rendering in the main thread on Mac & iOS #3166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sauwming merged 2 commits into from
Jul 7, 2022
Merged

Perform async rendering in the main thread on Mac & iOS #3166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
26c2f7f
Perform async rendering in the main thread on Mac & iOS
sauwming Jun 29, 2022
9c400f1
Remove unnecessary(?) locking
sauwming Jun 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 49 additions & 25 deletions pjmedia/src/pjmedia-videodev/darwin_dev.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ @interface VOutDelegate: NSObject
AVCaptureVideoPreviewLayer *prev_layer;

#if TARGET_OS_IPHONE
pj_bool_t is_running;
pj_bool_t is_rendering;
void *render_buf;
pj_size_t render_buf_size;
CGDataProviderRef render_data_provider;
Expand Down Expand Up @@ -512,9 +514,10 @@ static pj_status_t darwin_factory_default_param(pj_pool_t *pool,
@implementation VOutDelegate
#if TARGET_OS_IPHONE
- (void)update_image
{
{
NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();

CGImageRef cgIm = CGImageCreate(stream->size.w, stream->size.h,
8, 32, stream->bytes_per_row, colorSpace,
kCGImageAlphaFirst |
Expand All @@ -527,7 +530,16 @@ - (void)update_image
CGImageRelease(cgIm);

[pool release];
stream->is_rendering = PJ_FALSE;
}

- (void)finish_render
{
/* Do nothing. This function is serialized in the main thread, so when
* it is called, we can be sure that update_image() has completed.
*/
}

#endif

- (void)session_runtime_error:(NSNotification *)notification
Expand Down Expand Up @@ -1311,14 +1323,14 @@ static pj_status_t darwin_stream_start(pjmedia_vid_dev_stream *strm)
{
struct darwin_stream *stream = (struct darwin_stream*)strm;

PJ_UNUSED_ARG(stream);

PJ_LOG(4, (THIS_FILE, "Starting Darwin video stream"));

stream->is_running = PJ_TRUE;

if (stream->cap_session) {
dispatch_sync_on_main_queue(^{
[stream->cap_session startRunning];
});
[stream->cap_session
performSelectorOnMainThread:@selector(startRunning)
withObject:nil waitUntilDone:YES];

if (![stream->cap_session isRunning]) {
/* More info about the error should be reported in
Expand Down Expand Up @@ -1346,16 +1358,23 @@ static pj_status_t darwin_stream_put_frame(pjmedia_vid_dev_stream *strm,
*/
if (frame->size==0 || frame->buf==NULL)
return PJ_SUCCESS;


if (!stream->is_running)
return PJ_EINVALIDOP;

/* Prevent more than one async rendering task. */
if (stream->is_rendering)
return PJ_EIGNORED;

if (stream->frame_size >= frame->size)
pj_memcpy(stream->render_buf, frame->buf, frame->size);
else
pj_memcpy(stream->render_buf, frame->buf, stream->frame_size);

/* Perform video display in a background thread */
dispatch_sync_on_main_queue(^{
[stream->vout_delegate update_image];
});
/* Perform video display in the main thread */
stream->is_rendering = PJ_TRUE;
[stream->vout_delegate performSelectorOnMainThread:@selector(update_image)
withObject:nil waitUntilDone:NO];
#endif

return PJ_SUCCESS;
Expand All @@ -1371,10 +1390,16 @@ static pj_status_t darwin_stream_stop(pjmedia_vid_dev_stream *strm)

PJ_LOG(4, (THIS_FILE, "Stopping Darwin video stream"));

dispatch_sync_on_main_queue(^{
[stream->cap_session stopRunning];
});
[stream->cap_session performSelectorOnMainThread:@selector(stopRunning)
withObject:nil waitUntilDone:YES];
stream->has_image = PJ_FALSE;
stream->is_running = PJ_FALSE;

#if TARGET_OS_IPHONE
/* Wait until the rendering finishes */
[stream->vout_delegate performSelectorOnMainThread:@selector(finish_render)
withObject:nil waitUntilDone:YES];
#endif

return PJ_SUCCESS;
}
Expand Down Expand Up @@ -1411,27 +1436,26 @@ static pj_status_t darwin_stream_destroy(pjmedia_vid_dev_stream *strm)

#if TARGET_OS_IPHONE
if (stream->prev_layer) {
CALayer *prev_layer = stream->prev_layer;
dispatch_sync_on_main_queue(^{
[prev_layer removeFromSuperlayer];
[prev_layer release];
});
[stream->prev_layer
performSelectorOnMainThread:@selector(removeFromSuperlayer)
withObject:nil waitUntilDone:YES];
[stream->prev_layer release];
stream->prev_layer = nil;
}

if (stream->render_view) {
UIView *view = stream->render_view;
dispatch_sync_on_main_queue(^{
[view removeFromSuperview];
[view release];
});
[stream->render_view
performSelectorOnMainThread:@selector(removeFromSuperview)
withObject:nil waitUntilDone:YES];

[stream->render_view release];
stream->render_view = nil;
}

if (stream->render_data_provider) {
CGDataProviderRelease(stream->render_data_provider);
stream->render_data_provider = nil;
}
}
#endif /* TARGET_OS_IPHONE */

if (stream->queue) {
Expand Down
51 changes: 46 additions & 5 deletions pjmedia/src/pjmedia-videodev/ios_opengl_dev.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,11 @@ @interface GLView : UIView
pj_timestamp frame_ts;
unsigned ts_inc;
pjmedia_rect_size vid_size;
const pjmedia_frame *frame;
unsigned frame_size;

pj_bool_t is_rendering;
void *render_buf;
unsigned render_buf_size;

gl_buffers *gl_buf;
GLView *gl_view;
Expand Down Expand Up @@ -229,9 +233,11 @@ - (void)render
}

pjmedia_vid_dev_opengl_draw(stream->gl_buf, stream->vid_size.w, stream->vid_size.h,
stream->frame->buf);
stream->render_buf);

[stream->ogl_context presentRenderbuffer:GL_RENDERBUFFER];

stream->is_rendering = PJ_FALSE;
}

- (void)finish_render
Expand Down Expand Up @@ -262,7 +268,9 @@ - (void)change_format
pjmedia_vid_dev_stream **p_vid_strm)
{
struct iosgl_stream *strm;
const pjmedia_video_format_info *vfi;
const pjmedia_video_format_detail *vfd;
pjmedia_video_apply_fmt_param vafp;
pj_status_t status = PJ_SUCCESS;
CGRect rect;

Expand Down Expand Up @@ -329,6 +337,17 @@ - (void)change_format
iosgl_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_ORIENTATION,
&param->orient);
}

vfi = pjmedia_get_video_format_info(NULL, param->fmt.id);
if (!vfi) return PJMEDIA_EVID_BADFORMAT;
vafp.size = param->fmt.det.vid.size;
vafp.buffer = NULL;
if (vfi->apply_fmt(vfi, &vafp) != PJ_SUCCESS)
return PJMEDIA_EVID_BADFORMAT;

strm->frame_size = vafp.framebytes;
strm->render_buf_size = strm->frame_size;
strm->render_buf = pj_pool_alloc(strm->pool, strm->render_buf_size);

PJ_LOG(4, (THIS_FILE, "iOS OpenGL ES renderer successfully created"));

Expand Down Expand Up @@ -396,6 +415,7 @@ static pj_status_t iosgl_stream_set_cap(pjmedia_vid_dev_stream *s,

if (cap==PJMEDIA_VID_DEV_CAP_FORMAT) {
const pjmedia_video_format_info *vfi;
pjmedia_video_apply_fmt_param vafp;
pjmedia_format *fmt = (pjmedia_format *)pval;
iosgl_fmt_info *ifi;

Expand All @@ -406,8 +426,20 @@ static pj_status_t iosgl_stream_set_cap(pjmedia_vid_dev_stream *s,
fmt->id);
if (!vfi)
return PJMEDIA_EVID_BADFORMAT;

vafp.size = fmt->det.vid.size;
vafp.buffer = NULL;
if (vfi->apply_fmt(vfi, &vafp) != PJ_SUCCESS)
return PJMEDIA_EVID_BADFORMAT;

pjmedia_format_copy(&strm->param.fmt, fmt);

strm->frame_size = vafp.framebytes;
if (strm->render_buf_size < strm->frame_size) {
/* Realloc only when needed */
strm->render_buf_size = strm->frame_size;
strm->render_buf=pj_pool_alloc(strm->pool, strm->render_buf_size);
}

[strm->gl_view performSelectorOnMainThread:@selector(change_format)
withObject:nil waitUntilDone:YES];
Expand Down Expand Up @@ -480,10 +512,19 @@ static pj_status_t iosgl_stream_put_frame(pjmedia_vid_dev_stream *strm,
if (!stream->is_running)
return PJ_EINVALIDOP;

stream->frame = frame;
/* Prevent more than one async rendering task. */
if (stream->is_rendering)
return PJ_EIGNORED;

if (stream->frame_size >= frame->size)
pj_memcpy(stream->render_buf, frame->buf, frame->size);
else
pj_memcpy(stream->render_buf, frame->buf, stream->frame_size);

/* Perform OpenGL drawing in the main thread. */
stream->is_rendering = PJ_TRUE;
[stream->gl_view performSelectorOnMainThread:@selector(render)
withObject:nil waitUntilDone:YES];
withObject:nil waitUntilDone:NO];

return PJ_SUCCESS;
}
Expand Down Expand Up @@ -513,7 +554,7 @@ static pj_status_t iosgl_stream_destroy(pjmedia_vid_dev_stream *strm)

if (stream->is_running)
iosgl_stream_stop(strm);

if (stream->gl_view) {
[stream->gl_view performSelectorOnMainThread:@selector(deinit_gl)
withObject:nil waitUntilDone:YES];
Expand Down
1 change: 1 addition & 0 deletions pjsip-apps/src/pjsua/ios/ipjsua.xcodeproj/xcshareddata/xcschemes/ipjsua.xcscheme
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
ignoresPersistentStateOnLaunch = "NO"
debugDocumentVersioning = "YES"
debugServiceExtension = "internal"
enableGPUValidationMode = "1"
allowLocationSimulation = "YES">
<BuildableProductRunnable
runnableDebuggingMode = "0">
Expand Down