Skip to content

Save blend time and memory when rendering distant pieces of text #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JustAMan merged 7 commits into from
Mar 25, 2020
Merged

Save blend time and memory when rendering distant pieces of text #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8bc0001
Change binding part of BlendRenderResult
JustAMan Mar 24, 2020
0cc2bf8
Restructure renderBlend so it can return multiple pieces; JS side not…
JustAMan Mar 24, 2020
c08fd1d
Adapt JS side to renderBlend potentially returning a list of images
JustAMan Mar 24, 2020
efabc64
Fix build
JustAMan Mar 24, 2020
707a398
Fix skipping images when rendering a part
JustAMan Mar 24, 2020
efed854
Implement splitting by 3x3 grid
JustAMan Mar 24, 2020
a87c702
Fix issues found in review
JustAMan Mar 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
293 changes: 209 additions & 84 deletions src/SubtitleOctopus.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ class ReusableBuffer {
lessen_counter = 0;
return buffer;
}
int capacity() const {
return size;
}

private:
void *buffer;
Expand All @@ -79,18 +82,74 @@ const float MAX_UINT8_CAST = 255.9 / 255;

#define CLAMP_UINT8(value) ((value > MIN_UINT8_CAST) ? ((value < MAX_UINT8_CAST) ? (int)(value * 255) : 255) : 0)

typedef struct RenderBlendPart {
int dest_x, dest_y, dest_width, dest_height;
unsigned char *image;
RenderBlendPart *next;
} RenderBlendPart;

typedef struct {
int changed;
double blend_time;
int dest_x, dest_y, dest_width, dest_height;
unsigned char* image;
RenderBlendPart *part;
} RenderBlendResult;

// maximum regions - a grid of 3x3
#define MAX_BLEND_STORAGES (3 * 3)
typedef struct {
RenderBlendPart part;
ReusableBuffer buf;
bool taken;
} RenderBlendStorage;

typedef struct {
double eventFinish, emptyFinish;
int is_animated;
} EventStopTimesResult;

#define MIN(x, y) (((x) < (y)) ? (x) : (y))
#define MAX(x, y) (((x) > (y)) ? (x) : (y))

class BoundingBox {
public:
int min_x, max_x, min_y, max_y;

BoundingBox(): min_x(-1), max_x(-1), min_y(-1), max_y(-1) {}

bool empty() const {
return min_x == -1;
}

void add(int x1, int y1, int w, int h) {
int x2 = x1 + w - 1, y2 = y1 + h - 1;
min_x = (min_x < 0) ? x1 : MIN(min_x, x1);
min_y = (min_y < 0) ? y1 : MIN(min_y, y1);
max_x = (max_x < 0) ? x2 : MAX(max_x, x2);
max_y = (max_y < 0) ? y2 : MAX(max_y, y2);
}

bool intersets(const BoundingBox& other) const {
return !(other.min_x > max_x ||
other.max_x < min_x ||
other.min_y > max_y ||
other.max_y < min_y);
}

bool tryMerge(BoundingBox& other) {
if (!intersets(other)) return false;

min_x = MIN(min_x, other.min_x);
min_y = MIN(min_y, other.min_y);
max_x = MAX(max_x, other.max_x);
max_y = MAX(max_y, other.max_y);
return true;
}

void clear() {
min_x = max_x = min_y = max_y = -1;
}
};

static int _is_move_tag_animated(char *begin, char *end) {
int params[6];
int count = 0, value = 0, num_digits = 0;
Expand Down Expand Up @@ -376,98 +435,56 @@ class SubtitleOctopus {
}

double start_blend_time = emscripten_get_now();

// find bounding rect first
int min_x = img->dst_x, min_y = img->dst_y;
int max_x = img->dst_x + img->w - 1, max_y = img->dst_y + img->h - 1;
ASS_Image *cur;
for (cur = img->next; cur != NULL; cur = cur->next) {
if (cur->dst_x < min_x) min_x = cur->dst_x;
if (cur->dst_y < min_y) min_y = cur->dst_y;
int right = cur->dst_x + cur->w - 1;
int bottom = cur->dst_y + cur->h - 1;
if (right > max_x) max_x = right;
if (bottom > max_y) max_y = bottom;
for (int i = 0; i < MAX_BLEND_STORAGES; i++) {
m_blendParts[i].taken = false;
}

// make float buffer for blending
int width = max_x - min_x + 1, height = max_y - min_y + 1;
float* buf = (float*)m_blend.take(sizeof(float) * width * height * 4, 0);
if (buf == NULL) {
printf("libass: error: cannot allocate buffer for blending");
return NULL;
// split rendering region in 9 pieces (as on 3x3 grid)
int split_x_low = canvas_w / 3, split_x_high = 2 * canvas_w / 3;
int split_y_low = canvas_h / 3, split_y_high = 2 * canvas_h / 3;
BoundingBox boxes[MAX_BLEND_STORAGES];
for (ASS_Image *cur = img; cur != NULL; cur = cur->next) {
int index = 0;
int middle_x = cur->dst_x + (cur->w >> 1), middle_y = cur->dst_y + (cur->h >> 1);
if (middle_y > split_y_high) {
index += 2 * 3;
} else if (middle_y > split_y_low) {
index += 1 * 3;
}
if (middle_x > split_x_high) {
index += 2;
} else if (middle_y > split_x_low) {
index += 1;
}
boxes[index].add(cur->dst_x, cur->dst_y, cur->w, cur->h);
}
memset(buf, 0, sizeof(float) * width * height * 4);

// blend things in
for (cur = img; cur != NULL; cur = cur->next) {
int curw = cur->w, curh = cur->h;
if (curw == 0 || curh == 0) continue; // skip empty images
int a = (255 - (cur->color & 0xFF));
if (a == 0) continue; // skip transparent images

int curs = (cur->stride >= curw) ? cur->stride : curw;
int curx = cur->dst_x - min_x, cury = cur->dst_y - min_y;

unsigned char *bitmap = cur->bitmap;
float normalized_a = a / 255.0;
float r = ((cur->color >> 24) & 0xFF) / 255.0;
float g = ((cur->color >> 16) & 0xFF) / 255.0;
float b = ((cur->color >> 8) & 0xFF) / 255.0;

int buf_line_coord = cury * width;
for (int y = 0, bitmap_offset = 0; y < curh; y++, bitmap_offset += curs, buf_line_coord += width)
{
for (int x = 0; x < curw; x++)
{
float pix_alpha = bitmap[bitmap_offset + x] * normalized_a / 255.0;
float inv_alpha = 1.0 - pix_alpha;

int buf_coord = (buf_line_coord + curx + x) << 2;
float *buf_r = buf + buf_coord;
float *buf_g = buf + buf_coord + 1;
float *buf_b = buf + buf_coord + 2;
float *buf_a = buf + buf_coord + 3;

// do the compositing, pre-multiply image RGB with alpha for current pixel
*buf_a = pix_alpha + *buf_a * inv_alpha;
*buf_r = r * pix_alpha + *buf_r * inv_alpha;
*buf_g = g * pix_alpha + *buf_g * inv_alpha;
*buf_b = b * pix_alpha + *buf_b * inv_alpha;
// now merge regions as long as there are intersecting regions
for (;;) {
bool merged = false;
for (int box1 = 0; box1 < MAX_BLEND_STORAGES - 1; box1++) {
if (boxes[box1].empty()) continue;
for (int box2 = box1 + 1; box2 < MAX_BLEND_STORAGES; box2++) {
if (boxes[box2].empty()) continue;
if (boxes[box1].tryMerge(boxes[box2])) {
boxes[box2].clear();
merged = true;
}
}
}
if (!merged) break;
}

// now build the result;
// NOTE: we use a "view" over [float,float,float,float] array of pixels,
// so we _must_ go left-right top-bottom to not mangle the result
unsigned int *result = (unsigned int*)buf;
for (int y = 0, buf_line_coord = 0; y < height; y++, buf_line_coord += width) {
for (int x = 0; x < width; x++) {
unsigned int pixel = 0;
int buf_coord = (buf_line_coord + x) << 2;
float alpha = buf[buf_coord + 3];
if (alpha > MIN_UINT8_CAST) {
// need to un-multiply the result
float value = buf[buf_coord] / alpha;
pixel |= CLAMP_UINT8(value); // R
value = buf[buf_coord + 1] / alpha;
pixel |= CLAMP_UINT8(value) << 8; // G
value = buf[buf_coord + 2] / alpha;
pixel |= CLAMP_UINT8(value) << 16; // B
pixel |= CLAMP_UINT8(alpha) << 24; // A
}
result[buf_line_coord + x] = pixel;
}
m_blendResult.part = NULL;
for (int box = 0; box < MAX_BLEND_STORAGES; box++) {
if (boxes[box].empty()) continue;
RenderBlendPart *part = renderBlendPart(boxes[box], img);
if (part == NULL) return NULL;
part->next = m_blendResult.part;
m_blendResult.part = part;
}

// return the thing
m_blendResult.dest_x = min_x;
m_blendResult.dest_y = min_y;
m_blendResult.dest_width = width;
m_blendResult.dest_height = height;
m_blendResult.blend_time = emscripten_get_now() - start_blend_time;
m_blendResult.image = (unsigned char*)result;

return &m_blendResult;
}

Expand Down Expand Up @@ -562,8 +579,116 @@ class SubtitleOctopus {
}

private:
RenderBlendPart* renderBlendPart(const BoundingBox& rect, ASS_Image* img) {
// make float buffer for blending
int width = rect.max_x - rect.min_x + 1, height = rect.max_y - rect.min_y + 1;
float* buf = (float*)m_blend.take(sizeof(float) * width * height * 4, 0);
if (buf == NULL) {
printf("libass: error: cannot allocate buffer for blending");
return NULL;
}
memset(buf, 0, sizeof(float) * width * height * 4);

// blend things in
for (ASS_Image *cur = img; cur != NULL; cur = cur->next) {
if (cur->dst_x < rect.min_x || cur->dst_y < rect.min_y) continue; // skip images not fully within render region
int curw = cur->w, curh = cur->h;
if (curw == 0 || curh == 0 || cur->dst_x + curw - 1> rect.max_x || cur->dst_y + curh - 1 > rect.max_y) continue; // skip empty images or images outside render region
int a = (255 - (cur->color & 0xFF));
if (a == 0) continue; // skip transparent images

int curs = (cur->stride >= curw) ? cur->stride : curw;
int curx = cur->dst_x - rect.min_x, cury = cur->dst_y - rect.min_y;

unsigned char *bitmap = cur->bitmap;
float normalized_a = a / 255.0;
float r = ((cur->color >> 24) & 0xFF) / 255.0;
float g = ((cur->color >> 16) & 0xFF) / 255.0;
float b = ((cur->color >> 8) & 0xFF) / 255.0;

int buf_line_coord = cury * width;
for (int y = 0, bitmap_offset = 0; y < curh; y++, bitmap_offset += curs, buf_line_coord += width)
{
for (int x = 0; x < curw; x++)
{
float pix_alpha = bitmap[bitmap_offset + x] * normalized_a / 255.0;
float inv_alpha = 1.0 - pix_alpha;

int buf_coord = (buf_line_coord + curx + x) << 2;
float *buf_r = buf + buf_coord;
float *buf_g = buf + buf_coord + 1;
float *buf_b = buf + buf_coord + 2;
float *buf_a = buf + buf_coord + 3;

// do the compositing, pre-multiply image RGB with alpha for current pixel
*buf_a = pix_alpha + *buf_a * inv_alpha;
*buf_r = r * pix_alpha + *buf_r * inv_alpha;
*buf_g = g * pix_alpha + *buf_g * inv_alpha;
*buf_b = b * pix_alpha + *buf_b * inv_alpha;
}
}
}

// find closest free buffer
int needed = sizeof(unsigned int) * width * height;
RenderBlendStorage *storage = m_blendParts, *bigBuffer = NULL, *smallBuffer = NULL;
for (int buffer_index = 0; buffer_index < MAX_BLEND_STORAGES; buffer_index++, storage++) {
if (storage->taken) continue;
if (storage->buf.capacity() >= needed) {
if (bigBuffer == NULL || bigBuffer->buf.capacity() > storage->buf.capacity()) bigBuffer = storage;
} else {
if (smallBuffer == NULL || smallBuffer->buf.capacity() > storage->buf.capacity()) smallBuffer = storage;
}
}

if (bigBuffer != NULL) {
storage = bigBuffer;
} else if (smallBuffer != NULL) {
storage = smallBuffer;
} else {
printf("libass: cannot get a buffer for rendering part!\n");
return NULL;
}

unsigned int *result = (unsigned int*)storage->buf.take(needed, false);
if (result == NULL) {
printf("libass: cannot make a buffer for rendering part!\n");
return NULL;
}
storage->taken = true;

// now build the result;
for (int y = 0, buf_line_coord = 0; y < height; y++, buf_line_coord += width) {
for (int x = 0; x < width; x++) {
unsigned int pixel = 0;
int buf_coord = (buf_line_coord + x) << 2;
float alpha = buf[buf_coord + 3];
if (alpha > MIN_UINT8_CAST) {
// need to un-multiply the result
float value = buf[buf_coord] / alpha;
pixel |= CLAMP_UINT8(value); // R
value = buf[buf_coord + 1] / alpha;
pixel |= CLAMP_UINT8(value) << 8; // G
value = buf[buf_coord + 2] / alpha;
pixel |= CLAMP_UINT8(value) << 16; // B
pixel |= CLAMP_UINT8(alpha) << 24; // A
}
result[buf_line_coord + x] = pixel;
}
}

// return the thing
storage->part.dest_x = rect.min_x;
storage->part.dest_y = rect.min_y;
storage->part.dest_width = width;
storage->part.dest_height = height;
storage->part.image = (unsigned char*)result;
return &storage->part;
}

ReusableBuffer m_blend;
RenderBlendResult m_blendResult;
RenderBlendStorage m_blendParts[MAX_BLEND_STORAGES];
int *m_is_event_animated;
bool m_drop_animations;
};
Expand Down
12 changes: 9 additions & 3 deletions src/SubtitleOctopus.idl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,14 +155,20 @@ interface libass {
};

[NoDelete]
interface RenderBlendResult {
attribute long changed;
attribute double blend_time;
interface RenderBlendPart {
attribute long dest_x;
attribute long dest_y;
attribute long dest_width;
attribute long dest_height;
attribute ByteString image;
attribute RenderBlendPart next;
};

[NoDelete]
interface RenderBlendResult {
attribute long changed;
attribute double blend_time;
attribute RenderBlendPart part;
};

[NoDelete]
Expand Down
9 changes: 5 additions & 4 deletions src/post-worker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,10 +211,11 @@ self.blendRenderTiming = function (timing, force) {
var canvases = [], buffers = [];
if (renderResult.ptr != 0 && (renderResult.changed != 0 || force)) {
// make a copy, as we should free the memory so subsequent calls can utilize it
var result = new Uint8Array(HEAPU8.subarray(renderResult.image, renderResult.image + renderResult.dest_width * renderResult.dest_height * 4));

canvases = [{w: renderResult.dest_width, h: renderResult.dest_height, x: renderResult.dest_x, y: renderResult.dest_y, buffer: result.buffer}];
buffers = [result.buffer];
for (var part = renderResult.part; part.ptr != 0; part = part.next) {
var result = new Uint8Array(HEAPU8.subarray(part.image, part.image + part.dest_width * part.dest_height * 4));
canvases.push({w: part.dest_width, h: part.dest_height, x: part.dest_x, y: part.dest_y, buffer: result.buffer});
buffers.push(result.buffer);
}
}

return {
Expand Down