diff --git a/emcc b/emcc index 025daaf9dcecc..74b8905349dbf 100755 --- a/emcc +++ b/emcc @@ -1289,6 +1289,10 @@ try: # Emscripten logging.debug('LLVM => JS') extra_args = [] if not js_libraries else ['--libraries', ','.join(map(os.path.abspath, js_libraries))] + if memory_init_file: + shared.Settings.MEM_INIT_METHOD = 1 + else: + assert shared.Settings.MEM_INIT_METHOD != 1 final = shared.Building.emscripten(final, append_ext=False, extra_args=extra_args) if DEBUG: save_intermediate('original') @@ -1340,18 +1344,47 @@ try: js_transform_tempfiles = [final] - if memory_init_file: + if shared.Settings.MEM_INIT_METHOD > 0: memfile = target + '.mem' shared.try_delete(memfile) def repl(m): # handle chunking of the memory initializer - s = m.groups(0)[0] - if len(s) == 0 and not shared.Settings.EMTERPRETIFY: return m.group(0) # emterpreter must have a mem init file; otherwise, don't emit 0-size ones - open(memfile, 'wb').write(''.join(map(lambda x: chr(int(x or '0')), s.split(',')))) + s = m.group(1) + if len(s) == 0: return '' # don't emit 0-size ones + membytes = [int(x or '0') for x in s.split(',')] + while membytes and membytes[-1] == 0: + membytes.pop() + if not membytes: return '' + if not memory_init_file: + # memory initializer in a string literal + s = list(membytes) + if shared.Settings.ASSERTIONS: + # append checksum of length and content + crcTable = [] + for i in range(256): + crc = i + for bit in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0xedb88320) + crcTable.append(crc) + crc = 0xffffffff + n = len(s) + crc = crcTable[(crc ^ n) & 0xff] ^ (crc >> 8) + crc = crcTable[(crc ^ (n >> 8)) & 0xff] ^ (crc >> 8) + for i in s: + crc = crcTable[(crc ^ i) & 0xff] ^ (crc >> 8) + for i in range(4): + s.append((crc >> (8 * i)) & 0xff) + s = ''.join(map(chr, s)) + s = s.replace('\\', '\\\\').replace("'", "\\'") + s = s.replace('\n', '\\n').replace('\r', '\\r') + def escape(x): return '\\x{:02x}'.format(ord(x.group())) + s = re.sub('[\x80-\xff]', escape, s) + return "memoryInitializer = '%s';" % s + open(memfile, 'wb').write(''.join(map(chr, membytes))) if DEBUG: # Copy into temp dir as well, so can be run there too shared.safe_copy(memfile, os.path.join(shared.get_emscripten_temp_dir(), os.path.basename(memfile))) - return 'var memoryInitializer = "%s";' % os.path.basename(memfile) + return 'memoryInitializer = "%s";' % os.path.basename(memfile) src = re.sub(shared.JS.memory_initializer_pattern, repl, open(final).read(), count=1) open(final + '.mem.js', 'w').write(src) final += '.mem.js' diff --git a/src/postamble.js b/src/postamble.js index abc9510f2d649..989c4a5a508be 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -1,6 +1,35 @@ // === Auto-generated postamble setup entry stuff === +#if MEM_INIT_METHOD == 2 +#if USE_PTHREADS +if (memoryInitializer && !ENVIRONMENT_IS_PTHREAD) (function(s) { +#else +if (memoryInitializer) (function(s) { +#endif + var i, n = s.length; +#if ASSERTIONS + n -= 4; + var crc, bit, table = new Int32Array(256); + for (i = 0; i < 256; ++i) { + for (crc = i, bit = 0; bit < 8; ++bit) + crc = (crc >>> 1) ^ ((crc & 1) * 0xedb88320); + table[i] = crc >>> 0; + } + crc = -1; + crc = table[(crc ^ n) & 0xff] ^ (crc >>> 8); + crc = table[(crc ^ (n >>> 8)) & 0xff] ^ (crc >>> 8); + for (i = 0; i < s.length; ++i) { + crc = table[(crc ^ s.charCodeAt(i)) & 0xff] ^ (crc >>> 8); + } + assert(crc === 0, "memory initializer checksum"); +#endif + for (i = 0; i < n; ++i) { + HEAPU8[STATIC_BASE + i] = s.charCodeAt(i); + } +})(memoryInitializer); +#else +#if MEM_INIT_METHOD == 1 #if USE_PTHREADS if (memoryInitializer && !ENVIRONMENT_IS_PTHREAD) { #else @@ -52,6 +81,8 @@ if (memoryInitializer) { } } } +#endif +#endif function ExitStatus(status) { this.name = "ExitStatus"; diff --git a/src/settings.js b/src/settings.js index b4868d7fd2857..d7a637c636575 100644 --- a/src/settings.js +++ b/src/settings.js @@ -35,6 +35,11 @@ var INVOKE_RUN = 1; // Whether we will run the main() function. Disable if you e // can do with Module.callMain(), with an optional parameter of commandline args). var NO_EXIT_RUNTIME = 0; // If set, the runtime is not quit when main() completes (allowing code to // run afterwards, for example from the browser main event loop). +var MEM_INIT_METHOD = 0; // How to represent the initial memory content. + // 0: keep array literal representing the initial memory data + // 1: create a *.mem file containing the binary data of the initial memory; + // use the --memory-init-file command line switch to select this method + // 2: embed a string literal representing that initial memory data var TOTAL_STACK = 5*1024*1024; // The total stack size. There is no way to enlarge the stack, so this // value must be large enough for the program's requirements. If // assertions are on, we will assert on not exceeding this, otherwise, diff --git a/tests/meminit_pairs.c b/tests/meminit_pairs.c new file mode 100644 index 0000000000000..32f087c571369 --- /dev/null +++ b/tests/meminit_pairs.c @@ -0,0 +1,18 @@ +unsigned char problematic[] = { 0x20, 0x7c, 0x02, 0x07, 0x5f, 0xa0, 0xdf }; +int main() { + unsigned char a, b; + int result = 0, i, j; + for (i = 0; i < sizeof(problematic); ++i) { + a = problematic[i] ^ 32; + for (j = 0; j < sizeof(problematic); ++j) { + b = problematic[j] ^ 32; + if (((const unsigned char)data[a][2*b]) != a || + ((const unsigned char)data[a][2*b + 1]) != b) { + result = 1; + printf("data[0x%02x][0x%03x]=%x02x\n", a, 2*b, data[a][2*b]); + printf("data[0x%02x][0x%03x]=%x02x\n", a, 2*b + 1, data[a][2*b + 1]); + } + } + } + REPORT_RESULT() +} diff --git a/tests/parallel_test_core.py b/tests/parallel_test_core.py index 94e0ed26de1d6..98aca0450f5d2 100755 --- a/tests/parallel_test_core.py +++ b/tests/parallel_test_core.py @@ -14,7 +14,7 @@ assert not os.environ.get('EM_SAVE_DIR'), 'Need separate directories to avoid the parallel tests clashing' # run slower ones first, to optimize total time -optimal_order = ['asm3i', 'asm1i', 'asm2nn', 'asm3', 'asm2', 'asm2g', 'asm2f', 'asm1', 'default'] +optimal_order = ['asm3i', 'asm1i', 'asm2nn', 'asm3', 'asm2', 'asm2m', 'asm2g', 'asm2f', 'asm1', 'default'] assert set(optimal_order) == set(test_modes), 'need to update the list of slowest modes' # set up a background thread to report progress diff --git a/tests/runner.py b/tests/runner.py index 98336b7c76911..ea5d7747d8003 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -38,7 +38,7 @@ def path_from_root(*pathelems): # Core test runner class, shared between normal tests and benchmarks checked_sanity = False -test_modes = ['default', 'asm1', 'asm2', 'asm3', 'asm2f', 'asm2g', 'asm1i', 'asm3i', 'asm2nn'] +test_modes = ['default', 'asm1', 'asm2', 'asm3', 'asm2f', 'asm2g', 'asm1i', 'asm3i', 'asm2m', 'asm2nn'] test_index = 0 use_all_engines = os.environ.get('EM_ALL_ENGINES') # generally js engines are equivalent, testing 1 is enough. set this @@ -62,6 +62,14 @@ def skipme(self): # used by tests we ask on the commandline to be skipped, see r def is_emterpreter(self): return False + def uses_memory_init_file(self): + if self.emcc_args is None: + return None + elif '--memory-init-file' in self.emcc_args: + return int(self.emcc_args[self.emcc_args.index('--memory-init-file')+1]) + else: + return ('-O2' in self.emcc_args or '-O3' in self.emcc_args or '-Oz' in self.emcc_args) and not Settings.SIDE_MODULE + def setUp(self): Settings.reset() self.banned_js_engines = [] @@ -252,16 +260,10 @@ def build(self, src, dirname, filename, output_processor=None, main_file=None, a output_processor(open(filename + '.o.js').read()) if self.emcc_args is not None: - if '--memory-init-file' in self.emcc_args: - memory_init_file = int(self.emcc_args[self.emcc_args.index('--memory-init-file')+1]) - else: - memory_init_file = ('-O2' in self.emcc_args or '-O3' in self.emcc_args or '-Oz' in self.emcc_args) and not Settings.SIDE_MODULE src = open(filename + '.o.js').read() - if memory_init_file: + if self.uses_memory_init_file(): # side memory init file, or an empty one in the js assert ('/* memory initializer */' not in src) or ('/* memory initializer */ allocate([]' in src) - else: - assert 'memory initializer */' in src or '/*' not in src # memory initializer comment, or cleaned-up source with no comments def validate_asmjs(self, err): if 'uccessfully compiled asm.js code' in err and 'asm.js link error' not in err: diff --git a/tests/test_browser.py b/tests/test_browser.py index d38771e55d85c..05af5a12bc70d 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -2637,3 +2637,26 @@ def test_pthread_file_io(self): # Test that it is possible to send a signal via calling alarm(timeout), which in turn calls to the signal handler set by signal(SIGALRM, func); def test_sigalrm(self): self.btest(path_from_root('tests', 'sigalrm.cpp'), expected='0', args=['-O3']) + + def test_meminit_pairs(self): + d = 'const char *data[] = {\n "' + d += '",\n "'.join(''.join('\\x{:02x}\\x{:02x}'.format(i, j) + for j in range(256)) for i in range(256)) + with open(path_from_root('tests', 'meminit_pairs.c')) as f: + d += '"\n};\n' + f.read() + args = ["-O2", "--memory-init-file", "0", "-s", "MEM_INIT_METHOD=2", "-s", "ASSERTIONS=1"] + self.btest(d, expected='0', args=args + ["--closure", "0"]) + self.btest(d, expected='0', args=args + ["--closure", "0", "-g"]) + self.btest(d, expected='0', args=args + ["--closure", "1"]) + + def test_meminit_big(self): + d = 'const char *data[] = {\n "' + d += '",\n "'.join([''.join('\\x{:02x}\\x{:02x}'.format(i, j) + for j in range(256)) for i in range(256)]*256) + with open(path_from_root('tests', 'meminit_pairs.c')) as f: + d += '"\n};\n' + f.read() + assert len(d) > (1 << 27) # more than 32M memory initializer + args = ["-O2", "--memory-init-file", "0", "-s", "MEM_INIT_METHOD=2", "-s", "ASSERTIONS=1"] + self.btest(d, expected='0', args=args + ["--closure", "0"]) + self.btest(d, expected='0', args=args + ["--closure", "0", "-g"]) + self.btest(d, expected='0', args=args + ["--closure", "1"]) diff --git a/tests/test_core.py b/tests/test_core.py index be17f34be11d5..33e7a093bc2f7 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4374,6 +4374,13 @@ def test_strstr(self): self.do_run_from_file(src, output) def test_fnmatch(self): + # Run one test without assertions, for additional coverage + assert 'asm2m' in test_modes + if self.run_name == 'asm2m': + i = self.emcc_args.index('ASSERTIONS=1') + assert i > 0 and self.emcc_args[i-1] == '-s' + self.emcc_args[i] = 'ASSERTIONS=0' + test_path = path_from_root('tests', 'core', 'fnmatch') src, output = (test_path + s for s in ('.c', '.out')) self.do_run_from_file(src, output) @@ -4549,7 +4556,7 @@ def process(filename): try_delete(mem_file) self.do_run(src, ('size: 7\ndata: 100,-56,50,25,10,77,123\nloop: 100 -56 50 25 10 77 123 \ninput:hi there!\ntexto\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\n5 bytes to dev/null: 5\nok.\n \ntexte\n', 'size: 7\ndata: 100,-56,50,25,10,77,123\nloop: 100 -56 50 25 10 77 123 \ninput:hi there!\ntexto\ntexte\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\n5 bytes to dev/null: 5\nok.\n'), post_build=post, extra_emscripten_args=['-H', 'libc/fcntl.h']) - if '-O2' in self.emcc_args: + if self.uses_memory_init_file(): assert os.path.exists(mem_file) def test_files_m(self): @@ -7387,6 +7394,7 @@ def setUp(self): asm2g = make_run("asm2g", compiler=CLANG, emcc_args=["-O2", "-g", "-s", "ASSERTIONS=1", "-s", "SAFE_HEAP=1"]) asm1i = make_run("asm1i", compiler=CLANG, emcc_args=["-O1", '-s', 'EMTERPRETIFY=1']) asm3i = make_run("asm3i", compiler=CLANG, emcc_args=["-O3", '-s', 'EMTERPRETIFY=1']) +asm2m = make_run("asm2m", compiler=CLANG, emcc_args=["-O2", "--memory-init-file", "0", "-s", "MEM_INIT_METHOD=2", "-s", "ASSERTIONS=1"]) # Legacy test modes - asm2nn = make_run("asm2nn", compiler=CLANG, emcc_args=["-O2"], env={"EMCC_NATIVE_OPTIMIZER": "0"}) diff --git a/tests/test_other.py b/tests/test_other.py index 28f949eeb7fc9..9ac33f0aacb53 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -4879,3 +4879,17 @@ def test_debug_asmLastOpts(self): out, err = Popen([PYTHON, EMCC, 'src.c', '-s', 'EXPORTED_FUNCTIONS=["_main", "_treecount"]', '--minify', '0', '-g4', '-Oz']).communicate() self.assertContained('hello, world!', run_js('a.out.js')) + def test_meminit_crc(self): + with open('src.c', 'w') as f: + f.write(r''' +#include +int main() { printf("Mary had a little lamb.\n"); } +''') + out, err = Popen([PYTHON, EMCC, 'src.c', '-O2', '--memory-init-file', '0', '-s', 'MEM_INIT_METHOD=2', '-s', 'ASSERTIONS=1']).communicate() + with open('a.out.js', 'r') as f: + d = f.read() + d = d.replace('Mary had', 'Paul had') + with open('a.out.js', 'w') as f: + f.write(d) + out = run_js('a.out.js', assert_returncode=None, stderr=subprocess.STDOUT) + self.assertContained('Assertion failed: memory initializer checksum', out)