diff options
Diffstat (limited to 'auto_update_tests.py')
| -rwxr-xr-x | auto_update_tests.py | 353 |
1 files changed, 169 insertions, 184 deletions
diff --git a/auto_update_tests.py b/auto_update_tests.py index 8d681d214..7da4da948 100755 --- a/auto_update_tests.py +++ b/auto_update_tests.py @@ -24,7 +24,7 @@ from scripts.test.support import run_command, split_wast, write_wast, node_test_ from scripts.test.shared import ( ASM2WASM, MOZJS, NODEJS, WASM_OPT, WASM_AS, WASM_DIS, WASM_CTOR_EVAL, WASM_REDUCE, WASM_METADCE, BINARYEN_INSTALL_DIR, - BINARYEN_JS, has_shell_timeout, options, requested + BINARYEN_JS, has_shell_timeout, options, requested, get_test_dir, get_tests ) from scripts.test import lld @@ -33,47 +33,47 @@ from scripts.test import wasm2js def update_asm_js_tests(): print('[ processing and updating testcases... ]\n') - for asm in sorted(os.listdir(options.binaryen_test)): - if asm.endswith('.asm.js'): - for precise in [0, 1, 2]: - for opts in [1, 0]: - cmd = ASM2WASM + [os.path.join(options.binaryen_test, asm)] - if 'threads' in asm: - cmd += ['--enable-threads'] - wasm = asm.replace('.asm.js', '.fromasm') - if not precise: - cmd += ['--trap-mode=allow', '--ignore-implicit-traps'] - wasm += '.imprecise' - elif precise == 2: - cmd += ['--trap-mode=clamp'] - wasm += '.clamp' - if not opts: - wasm += '.no-opts' - if precise: - cmd += ['-O0'] # test that -O0 does nothing - else: - cmd += ['-O'] - if 'debugInfo' in asm: - cmd += ['-g'] - if 'noffi' in asm: - cmd += ['--no-legalize-javascript-ffi'] - if precise and opts: - # test mem init importing - open('a.mem', 'wb').write(bytes(asm, 'utf-8')) - cmd += ['--mem-init=a.mem'] - if asm[0] == 'e': - cmd += ['--mem-base=1024'] - if '4GB' in asm: - cmd += ['--mem-max=4294967296'] - if 'i64' in asm or 'wasm-only' in asm or 'noffi' in asm: - cmd += ['--wasm-only'] - print(' '.join(cmd)) - actual = run_command(cmd) - with open(os.path.join(options.binaryen_test, wasm), 'w') as o: - o.write(actual) - if 'debugInfo' in asm: - cmd += ['--source-map', os.path.join(options.binaryen_test, wasm + '.map'), '-o', 'a.wasm'] - run_command(cmd) + for asm in get_tests(options.binaryen_test, ['.asm.js']): + basename = os.path.basename(asm) + for precise in [0, 1, 2]: + for opts in [1, 0]: + cmd = ASM2WASM + [asm] + if 'threads' in basename: + cmd += ['--enable-threads'] + wasm = asm.replace('.asm.js', '.fromasm') + if not precise: + cmd += ['--trap-mode=allow', '--ignore-implicit-traps'] + wasm += '.imprecise' + elif precise == 2: + cmd += ['--trap-mode=clamp'] + wasm += '.clamp' + if not opts: + wasm += '.no-opts' + if precise: + cmd += ['-O0'] # test that -O0 does nothing + else: + cmd += ['-O'] + if 'debugInfo' in basename: + cmd += ['-g'] + if 'noffi' in basename: + cmd += ['--no-legalize-javascript-ffi'] + if precise and opts: + # test mem init importing + open('a.mem', 'wb').write(bytes(basename, 'utf-8')) + cmd += ['--mem-init=a.mem'] + if basename[0] == 'e': + cmd += ['--mem-base=1024'] + if '4GB' in basename: + cmd += ['--mem-max=4294967296'] + if 'i64' in basename or 'wasm-only' in basename or 'noffi' in basename: + cmd += ['--wasm-only'] + print(' '.join(cmd)) + actual = run_command(cmd) + with open(os.path.join(options.binaryen_test, wasm), 'w') as o: + o.write(actual) + if 'debugInfo' in basename: + cmd += ['--source-map', os.path.join(options.binaryen_test, wasm + '.map'), '-o', 'a.wasm'] + run_command(cmd) def update_wasm_opt_tests(): @@ -84,121 +84,116 @@ def update_wasm_opt_tests(): open(wast, 'w').write(open('a.wast').read()) print('\n[ checking wasm-opt parsing & printing... ]\n') - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'print'))): - if t.endswith('.wast'): - print('..', t) - wasm = os.path.basename(t).replace('.wast', '') - cmd = WASM_OPT + [os.path.join(options.binaryen_test, 'print', t), '--print', '-all'] - print(' ', ' '.join(cmd)) - actual = subprocess.check_output(cmd) - print(cmd, actual) - with open(os.path.join(options.binaryen_test, 'print', wasm + '.txt'), 'wb') as o: - o.write(actual) - cmd = WASM_OPT + [os.path.join(options.binaryen_test, 'print', t), '--print-minified', '-all'] - print(' ', ' '.join(cmd)) - actual = subprocess.check_output(cmd) - with open(os.path.join(options.binaryen_test, 'print', wasm + '.minified.txt'), 'wb') as o: - o.write(actual) + for t in get_tests(get_test_dir('print'), ['.wast']): + print('..', os.path.basename(t)) + wasm = t.replace('.wast', '') + cmd = WASM_OPT + [t, '--print', '-all'] + print(' ', ' '.join(cmd)) + actual = subprocess.check_output(cmd) + print(cmd, actual) + with open(wasm + '.txt', 'wb') as o: + o.write(actual) + cmd = WASM_OPT + [t, '--print-minified', '-all'] + print(' ', ' '.join(cmd)) + actual = subprocess.check_output(cmd) + with open(wasm + '.minified.txt', 'wb') as o: + o.write(actual) print('\n[ checking wasm-opt passes... ]\n') - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'passes'))): - if t.endswith(('.wast', '.wasm')): - print('..', t) - binary = '.wasm' in t - base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') - passname = base - if passname.isdigit(): - passname = open(os.path.join(options.binaryen_test, 'passes', passname + '.passes')).read().strip() - opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] - t = os.path.join(options.binaryen_test, 'passes', t) - actual = '' - for module, asserts in split_wast(t): - assert len(asserts) == 0 - write_wast('split.wast', module) - cmd = WASM_OPT + opts + ['split.wast', '--print'] - actual += run_command(cmd) - with open(os.path.join(options.binaryen_test, 'passes', base + ('.bin' if binary else '') + '.txt'), 'w') as o: - o.write(actual) - if 'emit-js-wrapper' in t: - with open('a.js') as i: - with open(t + '.js', 'w') as o: - o.write(i.read()) - if 'emit-spec-wrapper' in t: - with open('a.wat') as i: - with open(t + '.wat', 'w') as o: - o.write(i.read()) + for t in get_tests(get_test_dir('passes'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + binary = t.endswith('.wasm') + base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') + passname = base + if passname.isdigit(): + passname = open(os.path.join(options.binaryen_test, 'passes', passname + '.passes')).read().strip() + opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] + actual = '' + for module, asserts in split_wast(t): + assert len(asserts) == 0 + write_wast('split.wast', module) + cmd = WASM_OPT + opts + ['split.wast', '--print'] + actual += run_command(cmd) + with open(os.path.join(options.binaryen_test, 'passes', base + ('.bin' if binary else '') + '.txt'), 'w') as o: + o.write(actual) + if 'emit-js-wrapper' in t: + with open('a.js') as i: + with open(t + '.js', 'w') as o: + o.write(i.read()) + if 'emit-spec-wrapper' in t: + with open('a.wat') as i: + with open(t + '.wat', 'w') as o: + o.write(i.read()) print('\n[ checking wasm-opt testcases... ]\n') - for t in os.listdir(options.binaryen_test): - if t.endswith('.wast') and not t.startswith('spec'): - print('..', t) - t = os.path.join(options.binaryen_test, t) - f = t + '.from-wast' - cmd = WASM_OPT + [t, '--print', '-all'] - actual = run_command(cmd) - actual = actual.replace('printing before:\n', '') - open(f, 'w').write(actual) + for t in get_tests(options.binaryen_test, ['.wast']): + print('..', os.path.basename(t)) + f = t + '.from-wast' + cmd = WASM_OPT + [t, '--print', '-all'] + actual = run_command(cmd) + actual = actual.replace('printing before:\n', '') + open(f, 'w').write(actual) print('\n[ checking wasm-opt debugInfo read-write... ]\n') - for t in os.listdir(options.binaryen_test): - if t.endswith('.fromasm') and 'debugInfo' in t: - print('..', t) - t = os.path.join(options.binaryen_test, t) - f = t + '.read-written' - run_command(WASM_AS + [t, '--source-map=a.map', '-o', 'a.wasm', '-g']) - run_command(WASM_OPT + ['a.wasm', '--input-source-map=a.map', '-o', 'b.wasm', '--output-source-map=b.map', '-g']) - actual = run_command(WASM_DIS + ['b.wasm', '--source-map=b.map']) - open(f, 'w').write(actual) + for t in get_tests(options.binaryen_test, ['.fromasm']): + if 'debugInfo' not in t: + continue + print('..', os.path.basename(t)) + f = t + '.read-written' + run_command(WASM_AS + [t, '--source-map=a.map', '-o', 'a.wasm', '-g']) + run_command(WASM_OPT + ['a.wasm', '--input-source-map=a.map', '-o', 'b.wasm', '--output-source-map=b.map', '-g']) + actual = run_command(WASM_DIS + ['b.wasm', '--source-map=b.map']) + open(f, 'w').write(actual) def update_bin_fmt_tests(): print('\n[ checking binary format testcases... ]\n') - for wast in sorted(os.listdir(options.binaryen_test)): - if wast.endswith('.wast') and wast not in []: # blacklist some known failures - for debug_info in [0, 1]: - cmd = WASM_AS + [os.path.join(options.binaryen_test, wast), '-o', 'a.wasm', '-all'] - if debug_info: - cmd += ['-g'] - print(' '.join(cmd)) - if os.path.exists('a.wasm'): - os.unlink('a.wasm') - subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert os.path.exists('a.wasm') - - cmd = WASM_DIS + ['a.wasm', '-o', 'a.wast'] - print(' '.join(cmd)) - if os.path.exists('a.wast'): - os.unlink('a.wast') - subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert os.path.exists('a.wast') - actual = open('a.wast').read() - binary_name = wast + '.fromBinary' - if not debug_info: - binary_name += '.noDebugInfo' - with open(os.path.join(options.binaryen_test, binary_name), 'w') as o: - o.write(actual) + for wast in get_tests(options.binaryen_test, ['.wast']): + for debug_info in [0, 1]: + cmd = WASM_AS + [wast, '-o', 'a.wasm', '-all'] + if debug_info: + cmd += ['-g'] + print(' '.join(cmd)) + if os.path.exists('a.wasm'): + os.unlink('a.wasm') + subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert os.path.exists('a.wasm') + + cmd = WASM_DIS + ['a.wasm', '-o', 'a.wast'] + print(' '.join(cmd)) + if os.path.exists('a.wast'): + os.unlink('a.wast') + subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert os.path.exists('a.wast') + actual = open('a.wast').read() + binary_file = wast + '.fromBinary' + if not debug_info: + binary_file += '.noDebugInfo' + with open(binary_file, 'w') as o: + o.write(actual) def update_example_tests(): print('\n[ checking example testcases... ]\n') - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'example'))): + for t in get_tests(get_test_dir('example')): + basename = os.path.basename(t) output_file = os.path.join(options.binaryen_bin, 'example') libdir = os.path.join(BINARYEN_INSTALL_DIR, 'lib') cmd = ['-I' + os.path.join(options.binaryen_root, 'src'), '-g', '-pthread', '-o', output_file] if t.endswith('.txt'): # check if there is a trace in the file, if so, we should build it - out = subprocess.Popen([os.path.join(options.binaryen_root, 'scripts', 'clean_c_api_trace.py'), os.path.join(options.binaryen_test, 'example', t)], stdout=subprocess.PIPE).communicate()[0] + out = subprocess.Popen([os.path.join(options.binaryen_root, 'scripts', 'clean_c_api_trace.py'), t], stdout=subprocess.PIPE).communicate()[0] if len(out) == 0: - print(' (no trace in ', t, ')') + print(' (no trace in ', basename, ')') continue - print(' (will check trace in ', t, ')') + print(' (will check trace in ', basename, ')') src = 'trace.cpp' with open(src, 'wb') as o: o.write(out) - expected = os.path.join(options.binaryen_test, 'example', t + '.txt') + expected = t + '.txt' else: - src = os.path.join(options.binaryen_test, 'example', t) - expected = os.path.join(options.binaryen_test, 'example', '.'.join(t.split('.')[:-1]) + '.txt') + src = t + expected = os.path.splitext(t)[0] + '.txt' if not src.endswith(('.c', '.cpp')): continue # build the C file separately @@ -212,7 +207,7 @@ def update_example_tests(): subprocess.check_call(extra) # Link against the binaryen C library DSO, using rpath cmd = ['example.o', '-L' + libdir, '-lbinaryen', '-Wl,-rpath,' + os.path.abspath(libdir)] + cmd - print(' ', t, src, expected) + print(' ', basename, src, expected) if os.environ.get('COMPILER_FLAGS'): for f in os.environ.get('COMPILER_FLAGS').split(' '): cmd.append(f) @@ -235,16 +230,14 @@ def update_example_tests(): def update_wasm_dis_tests(): print('\n[ checking wasm-dis on provided binaries... ]\n') - for t in os.listdir(options.binaryen_test): - if t.endswith('.wasm') and not t.startswith('spec'): - print('..', t) - t = os.path.join(options.binaryen_test, t) - cmd = WASM_DIS + [t] - if os.path.isfile(t + '.map'): - cmd += ['--source-map', t + '.map'] - actual = run_command(cmd) + for t in get_tests(options.binaryen_test, ['.wasm']): + print('..', os.path.basename(t)) + cmd = WASM_DIS + [t] + if os.path.isfile(t + '.map'): + cmd += ['--source-map', t + '.map'] + actual = run_command(cmd) - open(t + '.fromBinary', 'w').write(actual) + open(t + '.fromBinary', 'w').write(actual) def update_binaryen_js_tests(): @@ -258,76 +251,68 @@ def update_binaryen_js_tests(): print('\n[ checking binaryen.js testcases... ]\n') node_has_wasm = NODEJS and node_has_webassembly(NODEJS) - for s in sorted(os.listdir(os.path.join(options.binaryen_test, 'binaryen.js'))): - if not s.endswith('.js'): - continue - print(s) + for s in get_tests(get_test_dir('binaryen.js', ['.js'])): + basename = os.path.basename(s) + print(basename) f = open('a.js', 'w') f.write(open(BINARYEN_JS).read()) if NODEJS: f.write(node_test_glue()) - test_path = os.path.join(options.binaryen_test, 'binaryen.js', s) - test_src = open(test_path).read() + test_src = open(s).read() f.write(test_src) f.close() if MOZJS or node_has_wasm or 'WebAssembly.' not in test_src: cmd = [MOZJS or NODEJS, 'a.js'] - if 'fatal' not in s: + if 'fatal' not in basename: out = run_command(cmd, stderr=subprocess.STDOUT) else: # expect an error - the specific error code will depend on the vm out = run_command(cmd, stderr=subprocess.STDOUT, expected_status=None) - with open(os.path.join(options.binaryen_test, 'binaryen.js', s + '.txt'), 'w') as o: + with open(s + '.txt', 'w') as o: o.write(out) else: - print('Skipping ' + test_path + ' because WebAssembly might not be supported') + print('Skipping ' + basename + ' because WebAssembly might not be supported') def update_ctor_eval_tests(): print('\n[ checking wasm-ctor-eval... ]\n') - for t in os.listdir(os.path.join(options.binaryen_test, 'ctor-eval')): - if t.endswith(('.wast', '.wasm')): - print('..', t) - t = os.path.join(options.binaryen_test, 'ctor-eval', t) - ctors = open(t + '.ctors').read().strip() - cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] - run_command(cmd) - actual = open('a.wast').read() - out = t + '.out' - with open(out, 'w') as o: - o.write(actual) + for t in get_tests(get_test_dir('ctor-eval'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + ctors = open(t + '.ctors').read().strip() + cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] + run_command(cmd) + actual = open('a.wast').read() + out = t + '.out' + with open(out, 'w') as o: + o.write(actual) def update_metadce_tests(): print('\n[ checking wasm-metadce... ]\n') - for t in os.listdir(os.path.join(options.binaryen_test, 'metadce')): - if t.endswith(('.wast', '.wasm')): - print('..', t) - t = os.path.join(options.binaryen_test, 'metadce', t) - graph = t + '.graph.txt' - cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S', '-all'] - stdout = run_command(cmd) - actual = open('a.wast').read() - out = t + '.dced' - with open(out, 'w') as o: - o.write(actual) - with open(out + '.stdout', 'w') as o: - o.write(stdout) + for t in get_tests(get_test_dir('metadce'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + graph = t + '.graph.txt' + cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S', '-all'] + stdout = run_command(cmd) + actual = open('a.wast').read() + out = t + '.dced' + with open(out, 'w') as o: + o.write(actual) + with open(out + '.stdout', 'w') as o: + o.write(stdout) def update_reduce_tests(): if not has_shell_timeout(): return print('\n[ checking wasm-reduce ]\n') - for t in os.listdir(os.path.join(options.binaryen_test, 'reduce')): - if t.endswith('.wast'): - print('..', t) - t = os.path.join(options.binaryen_test, 'reduce', t) - # convert to wasm - run_command(WASM_AS + [t, '-o', 'a.wasm']) - print(run_command(WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec' % WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm'])) - expected = t + '.txt' - run_command(WASM_DIS + ['c.wasm', '-o', expected]) + for t in get_tests(get_test_dir('reduce'), ['.wast']): + print('..', os.path.basename(t)) + # convert to wasm + run_command(WASM_AS + [t, '-o', 'a.wasm']) + print(run_command(WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec' % WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm'])) + expected = t + '.txt' + run_command(WASM_DIS + ['c.wasm', '-o', expected]) TEST_SUITES = OrderedDict([ |