diff options
| -rwxr-xr-x | auto_update_tests.py | 353 | ||||
| -rwxr-xr-x | check.py | 485 | ||||
| -rwxr-xr-x | scripts/test/asm2wasm.py | 24 | ||||
| -rwxr-xr-x | scripts/test/lld.py | 6 | ||||
| -rw-r--r-- | scripts/test/shared.py | 24 | ||||
| -rwxr-xr-x | scripts/test/wasm2js.py | 38 |
6 files changed, 444 insertions, 486 deletions
diff --git a/auto_update_tests.py b/auto_update_tests.py index 8d681d214..7da4da948 100755 --- a/auto_update_tests.py +++ b/auto_update_tests.py @@ -24,7 +24,7 @@ from scripts.test.support import run_command, split_wast, write_wast, node_test_ from scripts.test.shared import ( ASM2WASM, MOZJS, NODEJS, WASM_OPT, WASM_AS, WASM_DIS, WASM_CTOR_EVAL, WASM_REDUCE, WASM_METADCE, BINARYEN_INSTALL_DIR, - BINARYEN_JS, has_shell_timeout, options, requested + BINARYEN_JS, has_shell_timeout, options, requested, get_test_dir, get_tests ) from scripts.test import lld @@ -33,47 +33,47 @@ from scripts.test import wasm2js def update_asm_js_tests(): print('[ processing and updating testcases... ]\n') - for asm in sorted(os.listdir(options.binaryen_test)): - if asm.endswith('.asm.js'): - for precise in [0, 1, 2]: - for opts in [1, 0]: - cmd = ASM2WASM + [os.path.join(options.binaryen_test, asm)] - if 'threads' in asm: - cmd += ['--enable-threads'] - wasm = asm.replace('.asm.js', '.fromasm') - if not precise: - cmd += ['--trap-mode=allow', '--ignore-implicit-traps'] - wasm += '.imprecise' - elif precise == 2: - cmd += ['--trap-mode=clamp'] - wasm += '.clamp' - if not opts: - wasm += '.no-opts' - if precise: - cmd += ['-O0'] # test that -O0 does nothing - else: - cmd += ['-O'] - if 'debugInfo' in asm: - cmd += ['-g'] - if 'noffi' in asm: - cmd += ['--no-legalize-javascript-ffi'] - if precise and opts: - # test mem init importing - open('a.mem', 'wb').write(bytes(asm, 'utf-8')) - cmd += ['--mem-init=a.mem'] - if asm[0] == 'e': - cmd += ['--mem-base=1024'] - if '4GB' in asm: - cmd += ['--mem-max=4294967296'] - if 'i64' in asm or 'wasm-only' in asm or 'noffi' in asm: - cmd += ['--wasm-only'] - print(' '.join(cmd)) - actual = run_command(cmd) - with open(os.path.join(options.binaryen_test, wasm), 'w') as o: - o.write(actual) - if 'debugInfo' in asm: - cmd += ['--source-map', os.path.join(options.binaryen_test, wasm + '.map'), '-o', 'a.wasm'] - run_command(cmd) + for asm in get_tests(options.binaryen_test, ['.asm.js']): + basename = os.path.basename(asm) + for precise in [0, 1, 2]: + for opts in [1, 0]: + cmd = ASM2WASM + [asm] + if 'threads' in basename: + cmd += ['--enable-threads'] + wasm = asm.replace('.asm.js', '.fromasm') + if not precise: + cmd += ['--trap-mode=allow', '--ignore-implicit-traps'] + wasm += '.imprecise' + elif precise == 2: + cmd += ['--trap-mode=clamp'] + wasm += '.clamp' + if not opts: + wasm += '.no-opts' + if precise: + cmd += ['-O0'] # test that -O0 does nothing + else: + cmd += ['-O'] + if 'debugInfo' in basename: + cmd += ['-g'] + if 'noffi' in basename: + cmd += ['--no-legalize-javascript-ffi'] + if precise and opts: + # test mem init importing + open('a.mem', 'wb').write(bytes(basename, 'utf-8')) + cmd += ['--mem-init=a.mem'] + if basename[0] == 'e': + cmd += ['--mem-base=1024'] + if '4GB' in basename: + cmd += ['--mem-max=4294967296'] + if 'i64' in basename or 'wasm-only' in basename or 'noffi' in basename: + cmd += ['--wasm-only'] + print(' '.join(cmd)) + actual = run_command(cmd) + with open(os.path.join(options.binaryen_test, wasm), 'w') as o: + o.write(actual) + if 'debugInfo' in basename: + cmd += ['--source-map', os.path.join(options.binaryen_test, wasm + '.map'), '-o', 'a.wasm'] + run_command(cmd) def update_wasm_opt_tests(): @@ -84,121 +84,116 @@ def update_wasm_opt_tests(): open(wast, 'w').write(open('a.wast').read()) print('\n[ checking wasm-opt parsing & printing... ]\n') - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'print'))): - if t.endswith('.wast'): - print('..', t) - wasm = os.path.basename(t).replace('.wast', '') - cmd = WASM_OPT + [os.path.join(options.binaryen_test, 'print', t), '--print', '-all'] - print(' ', ' '.join(cmd)) - actual = subprocess.check_output(cmd) - print(cmd, actual) - with open(os.path.join(options.binaryen_test, 'print', wasm + '.txt'), 'wb') as o: - o.write(actual) - cmd = WASM_OPT + [os.path.join(options.binaryen_test, 'print', t), '--print-minified', '-all'] - print(' ', ' '.join(cmd)) - actual = subprocess.check_output(cmd) - with open(os.path.join(options.binaryen_test, 'print', wasm + '.minified.txt'), 'wb') as o: - o.write(actual) + for t in get_tests(get_test_dir('print'), ['.wast']): + print('..', os.path.basename(t)) + wasm = t.replace('.wast', '') + cmd = WASM_OPT + [t, '--print', '-all'] + print(' ', ' '.join(cmd)) + actual = subprocess.check_output(cmd) + print(cmd, actual) + with open(wasm + '.txt', 'wb') as o: + o.write(actual) + cmd = WASM_OPT + [t, '--print-minified', '-all'] + print(' ', ' '.join(cmd)) + actual = subprocess.check_output(cmd) + with open(wasm + '.minified.txt', 'wb') as o: + o.write(actual) print('\n[ checking wasm-opt passes... ]\n') - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'passes'))): - if t.endswith(('.wast', '.wasm')): - print('..', t) - binary = '.wasm' in t - base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') - passname = base - if passname.isdigit(): - passname = open(os.path.join(options.binaryen_test, 'passes', passname + '.passes')).read().strip() - opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] - t = os.path.join(options.binaryen_test, 'passes', t) - actual = '' - for module, asserts in split_wast(t): - assert len(asserts) == 0 - write_wast('split.wast', module) - cmd = WASM_OPT + opts + ['split.wast', '--print'] - actual += run_command(cmd) - with open(os.path.join(options.binaryen_test, 'passes', base + ('.bin' if binary else '') + '.txt'), 'w') as o: - o.write(actual) - if 'emit-js-wrapper' in t: - with open('a.js') as i: - with open(t + '.js', 'w') as o: - o.write(i.read()) - if 'emit-spec-wrapper' in t: - with open('a.wat') as i: - with open(t + '.wat', 'w') as o: - o.write(i.read()) + for t in get_tests(get_test_dir('passes'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + binary = t.endswith('.wasm') + base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') + passname = base + if passname.isdigit(): + passname = open(os.path.join(options.binaryen_test, 'passes', passname + '.passes')).read().strip() + opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] + actual = '' + for module, asserts in split_wast(t): + assert len(asserts) == 0 + write_wast('split.wast', module) + cmd = WASM_OPT + opts + ['split.wast', '--print'] + actual += run_command(cmd) + with open(os.path.join(options.binaryen_test, 'passes', base + ('.bin' if binary else '') + '.txt'), 'w') as o: + o.write(actual) + if 'emit-js-wrapper' in t: + with open('a.js') as i: + with open(t + '.js', 'w') as o: + o.write(i.read()) + if 'emit-spec-wrapper' in t: + with open('a.wat') as i: + with open(t + '.wat', 'w') as o: + o.write(i.read()) print('\n[ checking wasm-opt testcases... ]\n') - for t in os.listdir(options.binaryen_test): - if t.endswith('.wast') and not t.startswith('spec'): - print('..', t) - t = os.path.join(options.binaryen_test, t) - f = t + '.from-wast' - cmd = WASM_OPT + [t, '--print', '-all'] - actual = run_command(cmd) - actual = actual.replace('printing before:\n', '') - open(f, 'w').write(actual) + for t in get_tests(options.binaryen_test, ['.wast']): + print('..', os.path.basename(t)) + f = t + '.from-wast' + cmd = WASM_OPT + [t, '--print', '-all'] + actual = run_command(cmd) + actual = actual.replace('printing before:\n', '') + open(f, 'w').write(actual) print('\n[ checking wasm-opt debugInfo read-write... ]\n') - for t in os.listdir(options.binaryen_test): - if t.endswith('.fromasm') and 'debugInfo' in t: - print('..', t) - t = os.path.join(options.binaryen_test, t) - f = t + '.read-written' - run_command(WASM_AS + [t, '--source-map=a.map', '-o', 'a.wasm', '-g']) - run_command(WASM_OPT + ['a.wasm', '--input-source-map=a.map', '-o', 'b.wasm', '--output-source-map=b.map', '-g']) - actual = run_command(WASM_DIS + ['b.wasm', '--source-map=b.map']) - open(f, 'w').write(actual) + for t in get_tests(options.binaryen_test, ['.fromasm']): + if 'debugInfo' not in t: + continue + print('..', os.path.basename(t)) + f = t + '.read-written' + run_command(WASM_AS + [t, '--source-map=a.map', '-o', 'a.wasm', '-g']) + run_command(WASM_OPT + ['a.wasm', '--input-source-map=a.map', '-o', 'b.wasm', '--output-source-map=b.map', '-g']) + actual = run_command(WASM_DIS + ['b.wasm', '--source-map=b.map']) + open(f, 'w').write(actual) def update_bin_fmt_tests(): print('\n[ checking binary format testcases... ]\n') - for wast in sorted(os.listdir(options.binaryen_test)): - if wast.endswith('.wast') and wast not in []: # blacklist some known failures - for debug_info in [0, 1]: - cmd = WASM_AS + [os.path.join(options.binaryen_test, wast), '-o', 'a.wasm', '-all'] - if debug_info: - cmd += ['-g'] - print(' '.join(cmd)) - if os.path.exists('a.wasm'): - os.unlink('a.wasm') - subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert os.path.exists('a.wasm') - - cmd = WASM_DIS + ['a.wasm', '-o', 'a.wast'] - print(' '.join(cmd)) - if os.path.exists('a.wast'): - os.unlink('a.wast') - subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert os.path.exists('a.wast') - actual = open('a.wast').read() - binary_name = wast + '.fromBinary' - if not debug_info: - binary_name += '.noDebugInfo' - with open(os.path.join(options.binaryen_test, binary_name), 'w') as o: - o.write(actual) + for wast in get_tests(options.binaryen_test, ['.wast']): + for debug_info in [0, 1]: + cmd = WASM_AS + [wast, '-o', 'a.wasm', '-all'] + if debug_info: + cmd += ['-g'] + print(' '.join(cmd)) + if os.path.exists('a.wasm'): + os.unlink('a.wasm') + subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert os.path.exists('a.wasm') + + cmd = WASM_DIS + ['a.wasm', '-o', 'a.wast'] + print(' '.join(cmd)) + if os.path.exists('a.wast'): + os.unlink('a.wast') + subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert os.path.exists('a.wast') + actual = open('a.wast').read() + binary_file = wast + '.fromBinary' + if not debug_info: + binary_file += '.noDebugInfo' + with open(binary_file, 'w') as o: + o.write(actual) def update_example_tests(): print('\n[ checking example testcases... ]\n') - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'example'))): + for t in get_tests(get_test_dir('example')): + basename = os.path.basename(t) output_file = os.path.join(options.binaryen_bin, 'example') libdir = os.path.join(BINARYEN_INSTALL_DIR, 'lib') cmd = ['-I' + os.path.join(options.binaryen_root, 'src'), '-g', '-pthread', '-o', output_file] if t.endswith('.txt'): # check if there is a trace in the file, if so, we should build it - out = subprocess.Popen([os.path.join(options.binaryen_root, 'scripts', 'clean_c_api_trace.py'), os.path.join(options.binaryen_test, 'example', t)], stdout=subprocess.PIPE).communicate()[0] + out = subprocess.Popen([os.path.join(options.binaryen_root, 'scripts', 'clean_c_api_trace.py'), t], stdout=subprocess.PIPE).communicate()[0] if len(out) == 0: - print(' (no trace in ', t, ')') + print(' (no trace in ', basename, ')') continue - print(' (will check trace in ', t, ')') + print(' (will check trace in ', basename, ')') src = 'trace.cpp' with open(src, 'wb') as o: o.write(out) - expected = os.path.join(options.binaryen_test, 'example', t + '.txt') + expected = t + '.txt' else: - src = os.path.join(options.binaryen_test, 'example', t) - expected = os.path.join(options.binaryen_test, 'example', '.'.join(t.split('.')[:-1]) + '.txt') + src = t + expected = os.path.splitext(t)[0] + '.txt' if not src.endswith(('.c', '.cpp')): continue # build the C file separately @@ -212,7 +207,7 @@ def update_example_tests(): subprocess.check_call(extra) # Link against the binaryen C library DSO, using rpath cmd = ['example.o', '-L' + libdir, '-lbinaryen', '-Wl,-rpath,' + os.path.abspath(libdir)] + cmd - print(' ', t, src, expected) + print(' ', basename, src, expected) if os.environ.get('COMPILER_FLAGS'): for f in os.environ.get('COMPILER_FLAGS').split(' '): cmd.append(f) @@ -235,16 +230,14 @@ def update_example_tests(): def update_wasm_dis_tests(): print('\n[ checking wasm-dis on provided binaries... ]\n') - for t in os.listdir(options.binaryen_test): - if t.endswith('.wasm') and not t.startswith('spec'): - print('..', t) - t = os.path.join(options.binaryen_test, t) - cmd = WASM_DIS + [t] - if os.path.isfile(t + '.map'): - cmd += ['--source-map', t + '.map'] - actual = run_command(cmd) + for t in get_tests(options.binaryen_test, ['.wasm']): + print('..', os.path.basename(t)) + cmd = WASM_DIS + [t] + if os.path.isfile(t + '.map'): + cmd += ['--source-map', t + '.map'] + actual = run_command(cmd) - open(t + '.fromBinary', 'w').write(actual) + open(t + '.fromBinary', 'w').write(actual) def update_binaryen_js_tests(): @@ -258,76 +251,68 @@ def update_binaryen_js_tests(): print('\n[ checking binaryen.js testcases... ]\n') node_has_wasm = NODEJS and node_has_webassembly(NODEJS) - for s in sorted(os.listdir(os.path.join(options.binaryen_test, 'binaryen.js'))): - if not s.endswith('.js'): - continue - print(s) + for s in get_tests(get_test_dir('binaryen.js', ['.js'])): + basename = os.path.basename(s) + print(basename) f = open('a.js', 'w') f.write(open(BINARYEN_JS).read()) if NODEJS: f.write(node_test_glue()) - test_path = os.path.join(options.binaryen_test, 'binaryen.js', s) - test_src = open(test_path).read() + test_src = open(s).read() f.write(test_src) f.close() if MOZJS or node_has_wasm or 'WebAssembly.' not in test_src: cmd = [MOZJS or NODEJS, 'a.js'] - if 'fatal' not in s: + if 'fatal' not in basename: out = run_command(cmd, stderr=subprocess.STDOUT) else: # expect an error - the specific error code will depend on the vm out = run_command(cmd, stderr=subprocess.STDOUT, expected_status=None) - with open(os.path.join(options.binaryen_test, 'binaryen.js', s + '.txt'), 'w') as o: + with open(s + '.txt', 'w') as o: o.write(out) else: - print('Skipping ' + test_path + ' because WebAssembly might not be supported') + print('Skipping ' + basename + ' because WebAssembly might not be supported') def update_ctor_eval_tests(): print('\n[ checking wasm-ctor-eval... ]\n') - for t in os.listdir(os.path.join(options.binaryen_test, 'ctor-eval')): - if t.endswith(('.wast', '.wasm')): - print('..', t) - t = os.path.join(options.binaryen_test, 'ctor-eval', t) - ctors = open(t + '.ctors').read().strip() - cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] - run_command(cmd) - actual = open('a.wast').read() - out = t + '.out' - with open(out, 'w') as o: - o.write(actual) + for t in get_tests(get_test_dir('ctor-eval'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + ctors = open(t + '.ctors').read().strip() + cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] + run_command(cmd) + actual = open('a.wast').read() + out = t + '.out' + with open(out, 'w') as o: + o.write(actual) def update_metadce_tests(): print('\n[ checking wasm-metadce... ]\n') - for t in os.listdir(os.path.join(options.binaryen_test, 'metadce')): - if t.endswith(('.wast', '.wasm')): - print('..', t) - t = os.path.join(options.binaryen_test, 'metadce', t) - graph = t + '.graph.txt' - cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S', '-all'] - stdout = run_command(cmd) - actual = open('a.wast').read() - out = t + '.dced' - with open(out, 'w') as o: - o.write(actual) - with open(out + '.stdout', 'w') as o: - o.write(stdout) + for t in get_tests(get_test_dir('metadce'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + graph = t + '.graph.txt' + cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S', '-all'] + stdout = run_command(cmd) + actual = open('a.wast').read() + out = t + '.dced' + with open(out, 'w') as o: + o.write(actual) + with open(out + '.stdout', 'w') as o: + o.write(stdout) def update_reduce_tests(): if not has_shell_timeout(): return print('\n[ checking wasm-reduce ]\n') - for t in os.listdir(os.path.join(options.binaryen_test, 'reduce')): - if t.endswith('.wast'): - print('..', t) - t = os.path.join(options.binaryen_test, 'reduce', t) - # convert to wasm - run_command(WASM_AS + [t, '-o', 'a.wasm']) - print(run_command(WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec' % WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm'])) - expected = t + '.txt' - run_command(WASM_DIS + ['c.wasm', '-o', expected]) + for t in get_tests(get_test_dir('reduce'), ['.wast']): + print('..', os.path.basename(t)) + # convert to wasm + run_command(WASM_AS + [t, '-o', 'a.wasm']) + print(run_command(WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec' % WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm'])) + expected = t + '.txt' + run_command(WASM_DIS + ['c.wasm', '-o', expected]) TEST_SUITES = OrderedDict([ @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob import os import shutil import subprocess @@ -27,9 +28,9 @@ from scripts.test.shared import ( WASM_CTOR_EVAL, WASM_OPT, WASM_SHELL, WASM_METADCE, WASM_DIS, WASM_REDUCE, binary_format_check, delete_from_orbit, fail, fail_with_error, fail_if_not_identical, fail_if_not_contained, has_vanilla_emcc, - has_vanilla_llvm, minify_check, options, tests, requested, warnings, + has_vanilla_llvm, minify_check, options, requested, warnings, has_shell_timeout, fail_if_not_identical_to_file, with_pass_debug, - validate_binary + validate_binary, get_test_dir, get_tests ) # For shared.num_failures. Cannot import directly because modifications made in @@ -101,175 +102,156 @@ def run_wasm_opt_tests(): print('\n[ checking wasm-opt passes... ]\n') - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'passes'))): - if t.endswith(('.wast', '.wasm')): - print('..', t) - binary = '.wasm' in t - base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') - passname = base - if passname.isdigit(): - passname = open(os.path.join(options.binaryen_test, 'passes', passname + '.passes')).read().strip() - opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] - t = os.path.join(options.binaryen_test, 'passes', t) - actual = '' - for module, asserts in split_wast(t): - assert len(asserts) == 0 - write_wast('split.wast', module) - cmd = WASM_OPT + opts + ['split.wast', '--print'] - curr = run_command(cmd) - actual += curr - # also check debug mode output is valid - debugged = run_command(cmd + ['--debug'], stderr=subprocess.PIPE) - fail_if_not_contained(actual, debugged) - - # also check pass-debug mode - def check(): - pass_debug = run_command(cmd) - fail_if_not_identical(curr, pass_debug) - with_pass_debug(check) - - expected_file = os.path.join(options.binaryen_test, 'passes', - base + ('.bin' if binary else '') + '.txt') - fail_if_not_identical_to_file(actual, expected_file) - - if 'emit-js-wrapper' in t: - with open('a.js') as actual: - fail_if_not_identical_to_file(actual.read(), t + '.js') - if 'emit-spec-wrapper' in t: - with open('a.wat') as actual: - fail_if_not_identical_to_file(actual.read(), t + '.wat') + for t in get_tests(get_test_dir('passes'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + binary = '.wasm' in t + base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') + passname = base + if passname.isdigit(): + passname = open(os.path.join(get_test_dir('passes'), passname + '.passes')).read().strip() + opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] + actual = '' + for module, asserts in split_wast(t): + assert len(asserts) == 0 + write_wast('split.wast', module) + cmd = WASM_OPT + opts + ['split.wast', '--print'] + curr = run_command(cmd) + actual += curr + # also check debug mode output is valid + debugged = run_command(cmd + ['--debug'], stderr=subprocess.PIPE) + fail_if_not_contained(actual, debugged) + + # also check pass-debug mode + def check(): + pass_debug = run_command(cmd) + fail_if_not_identical(curr, pass_debug) + with_pass_debug(check) + + expected_file = os.path.join(get_test_dir('passes'), base + ('.bin' if binary else '') + '.txt') + fail_if_not_identical_to_file(actual, expected_file) + + if 'emit-js-wrapper' in t: + with open('a.js') as actual: + fail_if_not_identical_to_file(actual.read(), t + '.js') + if 'emit-spec-wrapper' in t: + with open('a.wat') as actual: + fail_if_not_identical_to_file(actual.read(), t + '.wat') print('\n[ checking wasm-opt parsing & printing... ]\n') - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'print'))): - if t.endswith('.wast'): - print('..', t) - wasm = os.path.basename(t).replace('.wast', '') - cmd = WASM_OPT + [os.path.join(options.binaryen_test, 'print', t), '--print', '-all'] - print(' ', ' '.join(cmd)) - actual, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).communicate() - expected_file = os.path.join(options.binaryen_test, 'print', wasm + '.txt') - fail_if_not_identical_to_file(actual, expected_file) - cmd = WASM_OPT + [os.path.join(options.binaryen_test, 'print', t), '--print-minified', '-all'] - print(' ', ' '.join(cmd)) - actual, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).communicate() - fail_if_not_identical(actual.strip(), open(os.path.join(options.binaryen_test, 'print', wasm + '.minified.txt')).read().strip()) + for t in get_tests(get_test_dir('print'), ['.wast']): + print('..', os.path.basename(t)) + wasm = os.path.basename(t).replace('.wast', '') + cmd = WASM_OPT + [t, '--print', '-all'] + print(' ', ' '.join(cmd)) + actual, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).communicate() + expected_file = os.path.join(get_test_dir('print'), wasm + '.txt') + fail_if_not_identical_to_file(actual, expected_file) + cmd = WASM_OPT + [os.path.join(get_test_dir('print'), t), '--print-minified', '-all'] + print(' ', ' '.join(cmd)) + actual, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).communicate() + fail_if_not_identical(actual.strip(), open(os.path.join(get_test_dir('print'), wasm + '.minified.txt')).read().strip()) print('\n[ checking wasm-opt testcases... ]\n') - for t in tests: - if t.endswith('.wast') and not t.startswith('spec'): - print('..', t) - t = os.path.join(options.binaryen_test, t) - f = t + '.from-wast' - cmd = WASM_OPT + [t, '--print', '-all'] - actual = run_command(cmd) - actual = actual.replace('printing before:\n', '') + for t in get_tests(options.binaryen_test, ['.wast']): + print('..', os.path.basename(t)) + f = t + '.from-wast' + cmd = WASM_OPT + [t, '--print', '-all'] + actual = run_command(cmd) + actual = actual.replace('printing before:\n', '') - fail_if_not_identical_to_file(actual, f) + fail_if_not_identical_to_file(actual, f) - binary_format_check(t, wasm_as_args=['-g']) # test with debuginfo - binary_format_check(t, wasm_as_args=[], binary_suffix='.fromBinary.noDebugInfo') # test without debuginfo + binary_format_check(t, wasm_as_args=['-g']) # test with debuginfo + binary_format_check(t, wasm_as_args=[], binary_suffix='.fromBinary.noDebugInfo') # test without debuginfo - minify_check(t) + minify_check(t) print('\n[ checking wasm-opt debugInfo read-write... ]\n') - test_dir = os.path.join(options.binaryen_root, 'test') - for t in os.listdir(test_dir): - if t.endswith('.fromasm') and 'debugInfo' in t: - print('..', t) - t = os.path.join(test_dir, t) - f = t + '.read-written' - run_command(WASM_AS + [t, '--source-map=a.map', '-o', 'a.wasm', '-g']) - run_command(WASM_OPT + ['a.wasm', '--input-source-map=a.map', '-o', 'b.wasm', '--output-source-map=b.map', '-g']) - actual = run_command(WASM_DIS + ['b.wasm', '--source-map=b.map']) - fail_if_not_identical_to_file(actual, f) + for t in get_tests(options.binaryen_test, ['.fromasm']): + if 'debugInfo' not in t: + continue + print('..', os.path.basename(t)) + f = t + '.read-written' + run_command(WASM_AS + [t, '--source-map=a.map', '-o', 'a.wasm', '-g']) + run_command(WASM_OPT + ['a.wasm', '--input-source-map=a.map', '-o', 'b.wasm', '--output-source-map=b.map', '-g']) + actual = run_command(WASM_DIS + ['b.wasm', '--source-map=b.map']) + fail_if_not_identical_to_file(actual, f) def run_wasm_dis_tests(): print('\n[ checking wasm-dis on provided binaries... ]\n') - for t in tests: - if t.endswith('.wasm') and not t.startswith('spec'): - print('..', t) - t = os.path.join(options.binaryen_test, t) - cmd = WASM_DIS + [t] - if os.path.isfile(t + '.map'): - cmd += ['--source-map', t + '.map'] + for t in get_tests(options.binaryen_test, ['.wasm']): + print('..', os.path.basename(t)) + cmd = WASM_DIS + [t] + if os.path.isfile(t + '.map'): + cmd += ['--source-map', t + '.map'] - actual = run_command(cmd) - fail_if_not_identical_to_file(actual, t + '.fromBinary') + actual = run_command(cmd) + fail_if_not_identical_to_file(actual, t + '.fromBinary') - # also verify there are no validation errors - def check(): - cmd = WASM_OPT + [t, '-all'] - run_command(cmd) + # also verify there are no validation errors + def check(): + cmd = WASM_OPT + [t, '-all'] + run_command(cmd) - with_pass_debug(check) + with_pass_debug(check) - validate_binary(t) + validate_binary(t) def run_crash_tests(): print("\n[ checking we don't crash on tricky inputs... ]\n") - test_dir = os.path.join(options.binaryen_test, 'crash') - for t in os.listdir(test_dir): - if t.endswith(('.wast', '.wasm')): - print('..', t) - t = os.path.join(test_dir, t) - cmd = WASM_OPT + [t] - # expect a parse error to be reported - run_command(cmd, expected_err='parse exception:', err_contains=True, expected_status=1) + for t in get_tests(get_test_dir('crash'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + cmd = WASM_OPT + [t] + # expect a parse error to be reported + run_command(cmd, expected_err='parse exception:', err_contains=True, expected_status=1) def run_dylink_tests(): print("\n[ we emit dylink sections properly... ]\n") - for t in os.listdir(options.binaryen_test): - if t.startswith('dylib') and t.endswith('.wasm'): - print('..', t) - t = os.path.join(options.binaryen_test, t) - cmd = WASM_OPT + [t, '-o', 'a.wasm'] - run_command(cmd) - with open('a.wasm', 'rb') as output: - index = output.read().find(b'dylink') - print(' ', index) - assert index == 11, 'dylink section must be first, right after the magic number etc.' + dylink_tests = glob.glob(os.path.join(options.binaryen_test, 'dylib*.wasm')) + for t in sorted(dylink_tests): + print('..', os.path.basename(t)) + cmd = WASM_OPT + [t, '-o', 'a.wasm'] + run_command(cmd) + with open('a.wasm', 'rb') as output: + index = output.read().find(b'dylink') + print(' ', index) + assert index == 11, 'dylink section must be first, right after the magic number etc.' def run_ctor_eval_tests(): print('\n[ checking wasm-ctor-eval... ]\n') - test_dir = os.path.join(options.binaryen_test, 'ctor-eval') - for t in os.listdir(test_dir): - if t.endswith(('.wast', '.wasm')): - print('..', t) - t = os.path.join(test_dir, t) - ctors = open(t + '.ctors').read().strip() - cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] - run_command(cmd) - actual = open('a.wast').read() - out = t + '.out' - fail_if_not_identical_to_file(actual, out) + for t in get_tests(get_test_dir('ctor-eval'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + ctors = open(t + '.ctors').read().strip() + cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] + run_command(cmd) + actual = open('a.wast').read() + out = t + '.out' + fail_if_not_identical_to_file(actual, out) def run_wasm_metadce_tests(): print('\n[ checking wasm-metadce ]\n') - test_dir = os.path.join(options.binaryen_test, 'metadce') - for t in os.listdir(test_dir): - if t.endswith(('.wast', '.wasm')): - print('..', t) - t = os.path.join(test_dir, t) - graph = t + '.graph.txt' - cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S', '-all'] - stdout = run_command(cmd) - expected = t + '.dced' - with open('a.wast') as seen: - fail_if_not_identical_to_file(seen.read(), expected) - fail_if_not_identical_to_file(stdout, expected + '.stdout') + for t in get_tests(get_test_dir('metadce'), ['.wast', '.wasm']): + print('..', os.path.basename(t)) + graph = t + '.graph.txt' + cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S', '-all'] + stdout = run_command(cmd) + expected = t + '.dced' + with open('a.wast') as seen: + fail_if_not_identical_to_file(seen.read(), expected) + fail_if_not_identical_to_file(stdout, expected + '.stdout') def run_wasm_reduce_tests(): @@ -280,18 +262,15 @@ def run_wasm_reduce_tests(): print('\n[ checking wasm-reduce testcases]\n') # fixed testcases - test_dir = os.path.join(options.binaryen_test, 'reduce') - for t in os.listdir(test_dir): - if t.endswith('.wast'): - print('..', t) - t = os.path.join(test_dir, t) - # convert to wasm - run_command(WASM_AS + [t, '-o', 'a.wasm']) - run_command(WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec -all' % WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm', '--timeout=4']) - expected = t + '.txt' - run_command(WASM_DIS + ['c.wasm', '-o', 'a.wast']) - with open('a.wast') as seen: - fail_if_not_identical_to_file(seen.read(), expected) + for t in get_tests(get_test_dir('reduce'), ['.wast']): + print('..', os.path.basename(t)) + # convert to wasm + run_command(WASM_AS + [t, '-o', 'a.wasm']) + run_command(WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec -all' % WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm', '--timeout=4']) + expected = t + '.txt' + run_command(WASM_DIS + ['c.wasm', '-o', 'a.wast']) + with open('a.wast') as seen: + fail_if_not_identical_to_file(seen.read(), expected) # run on a nontrivial fuzz testcase, for general coverage # this is very slow in ThreadSanitizer, so avoid it there @@ -314,127 +293,125 @@ def run_spec_tests(): # FIXME we support old and new memory formats, for now, until 0xc, and so can't pass this old-style test. BLACKLIST = ['binary.wast'] # FIXME to update the spec to 0xd, we need to implement (register "name") for import.wast - spec_tests = [os.path.join('spec', t) for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'spec'))) if t not in BLACKLIST] + spec_tests = get_tests(get_test_dir('spec'), ['.wast']) + spec_tests = [t for t in spec_tests if os.path.basename(t) not in BLACKLIST] else: spec_tests = options.spec_tests[:] - for t in spec_tests: - if t.startswith('spec') and t.endswith('.wast'): - print('..', t) - wast = os.path.join(options.binaryen_test, t) + for wast in spec_tests: + print('..', os.path.basename(wast)) - # skip checks for some tests - if os.path.basename(wast) in ['linking.wast', 'nop.wast', 'stack.wast', 'typecheck.wast', 'unwind.wast']: # FIXME - continue + # skip checks for some tests + if os.path.basename(wast) in ['linking.wast', 'nop.wast', 'stack.wast', 'typecheck.wast', 'unwind.wast']: # FIXME + continue + + def run_spec_test(wast): + cmd = WASM_SHELL + [wast] + # we must skip the stack machine portions of spec tests or apply other extra args + extra = {} + cmd = cmd + (extra.get(os.path.basename(wast)) or []) + return run_command(cmd, stderr=subprocess.PIPE) + + def run_opt_test(wast): + # check optimization validation + cmd = WASM_OPT + [wast, '-O', '-all'] + run_command(cmd) - def run_spec_test(wast): - cmd = WASM_SHELL + [wast] - # we must skip the stack machine portions of spec tests or apply other extra args - extra = { - } - cmd = cmd + (extra.get(os.path.basename(wast)) or []) - return run_command(cmd, stderr=subprocess.PIPE) - - def run_opt_test(wast): - # check optimization validation - cmd = WASM_OPT + [wast, '-O', '-all'] - run_command(cmd) - - def check_expected(actual, expected): - if expected and os.path.exists(expected): - expected = open(expected).read() - - # fix it up, our pretty (i32.const 83) must become compared to a homely 83 : i32 - def fix_expected(x): - x = x.strip() - if not x: - return x - v, t = x.split(' : ') - if v.endswith('.'): - v = v[:-1] # remove trailing '.' - return '(' + t + '.const ' + v + ')' - - def fix_actual(x): - if '[trap ' in x: - return '' + def check_expected(actual, expected): + if expected and os.path.exists(expected): + expected = open(expected).read() + + # fix it up, our pretty (i32.const 83) must become compared to a homely 83 : i32 + def fix_expected(x): + x = x.strip() + if not x: return x + v, t = x.split(' : ') + if v.endswith('.'): + v = v[:-1] # remove trailing '.' + return '(' + t + '.const ' + v + ')' + + def fix_actual(x): + if '[trap ' in x: + return '' + return x + + expected = '\n'.join(map(fix_expected, expected.split('\n'))) + actual = '\n'.join(map(fix_actual, actual.split('\n'))) + print(' (using expected output)') + actual = actual.strip() + expected = expected.strip() + if actual != expected: + fail(actual, expected) + + expected = os.path.join(get_test_dir('spec'), 'expected-output', os.path.basename(wast) + '.log') + + # some spec tests should fail (actual process failure, not just assert_invalid) + try: + actual = run_spec_test(wast) + except Exception as e: + if ('wasm-validator error' in str(e) or 'parse exception' in str(e)) and '.fail.' in os.path.basename(wast): + print('<< test failed as expected >>') + continue # don't try all the binary format stuff TODO + else: + fail_with_error(str(e)) - expected = '\n'.join(map(fix_expected, expected.split('\n'))) - actual = '\n'.join(map(fix_actual, actual.split('\n'))) - print(' (using expected output)') - actual = actual.strip() - expected = expected.strip() - if actual != expected: - fail(actual, expected) - - expected = os.path.join(options.binaryen_test, 'spec', 'expected-output', os.path.basename(wast) + '.log') - - # some spec tests should fail (actual process failure, not just assert_invalid) - try: - actual = run_spec_test(wast) - except Exception as e: - if ('wasm-validator error' in str(e) or 'parse exception' in str(e)) and '.fail.' in t: - print('<< test failed as expected >>') - continue # don't try all the binary format stuff TODO - else: - fail_with_error(str(e)) - - check_expected(actual, expected) - - # skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature - if os.path.basename(wast) in ['exports.wast']: # FIXME - continue + check_expected(actual, expected) - # we must ignore some binary format splits - splits_to_skip = { - 'func.wast': [2], - 'return.wast': [2] - } - - # check binary format. here we can verify execution of the final - # result, no need for an output verification - # some wast files cannot be split: - # * comments.wast: contains characters that are not valid utf-8, - # so our string splitting code fails there - if os.path.basename(wast) not in ['comments.wast']: - split_num = 0 - actual = '' - for module, asserts in split_wast(wast): - skip = splits_to_skip.get(os.path.basename(wast)) or [] - if split_num in skip: - print(' skipping split module', split_num - 1) - split_num += 1 - continue - print(' testing split module', split_num) + # skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature + if os.path.basename(wast) in ['exports.wast']: # FIXME + continue + + # we must ignore some binary format splits + splits_to_skip = { + 'func.wast': [2], + 'return.wast': [2] + } + + # check binary format. here we can verify execution of the final + # result, no need for an output verification + # some wast files cannot be split: + # * comments.wast: contains characters that are not valid utf-8, + # so our string splitting code fails there + if os.path.basename(wast) not in ['comments.wast']: + split_num = 0 + actual = '' + for module, asserts in split_wast(wast): + skip = splits_to_skip.get(os.path.basename(wast)) or [] + if split_num in skip: + print(' skipping split module', split_num - 1) split_num += 1 - write_wast('split.wast', module, asserts) - run_spec_test('split.wast') # before binary stuff - just check it's still ok split out - run_opt_test('split.wast') # also that our optimizer doesn't break on it - result_wast = binary_format_check('split.wast', verify_final_result=False, original_wast=wast) - # add the asserts, and verify that the test still passes - open(result_wast, 'a').write('\n' + '\n'.join(asserts)) - actual += run_spec_test(result_wast) - # compare all the outputs to the expected output - check_expected(actual, os.path.join(options.binaryen_test, 'spec', 'expected-output', os.path.basename(wast) + '.log')) - else: - # handle unsplittable wast files - run_spec_test(wast) + continue + print(' testing split module', split_num) + split_num += 1 + write_wast('split.wast', module, asserts) + run_spec_test('split.wast') # before binary stuff - just check it's still ok split out + run_opt_test('split.wast') # also that our optimizer doesn't break on it + result_wast = binary_format_check('split.wast', verify_final_result=False, original_wast=wast) + # add the asserts, and verify that the test still passes + open(result_wast, 'a').write('\n' + '\n'.join(asserts)) + actual += run_spec_test(result_wast) + # compare all the outputs to the expected output + check_expected(actual, os.path.join(get_test_dir('spec'), 'expected-output', os.path.basename(wast) + '.log')) + else: + # handle unsplittable wast files + run_spec_test(wast) def run_validator_tests(): print('\n[ running validation tests... ]\n') # Ensure the tests validate by default - cmd = WASM_AS + [os.path.join(options.binaryen_test, 'validator', 'invalid_export.wast')] + cmd = WASM_AS + [os.path.join(get_test_dir('validator'), 'invalid_export.wast')] run_command(cmd) - cmd = WASM_AS + [os.path.join(options.binaryen_test, 'validator', 'invalid_import.wast')] + cmd = WASM_AS + [os.path.join(get_test_dir('validator'), 'invalid_import.wast')] run_command(cmd) - cmd = WASM_AS + ['--validate=web', os.path.join(options.binaryen_test, 'validator', 'invalid_export.wast')] + cmd = WASM_AS + ['--validate=web', os.path.join(get_test_dir('validator'), 'invalid_export.wast')] run_command(cmd, expected_status=1) - cmd = WASM_AS + ['--validate=web', os.path.join(options.binaryen_test, 'validator', 'invalid_import.wast')] + cmd = WASM_AS + ['--validate=web', os.path.join(get_test_dir('validator'), 'invalid_import.wast')] run_command(cmd, expected_status=1) - cmd = WASM_AS + ['--validate=none', os.path.join(options.binaryen_test, 'validator', 'invalid_return.wast')] + cmd = WASM_AS + ['--validate=none', os.path.join(get_test_dir('validator'), 'invalid_return.wast')] run_command(cmd) - cmd = WASM_AS + [os.path.join(options.binaryen_test, 'validator', 'invalid_number.wast')] + cmd = WASM_AS + [os.path.join(get_test_dir('validator'), 'invalid_number.wast')] run_command(cmd, expected_status=1) @@ -459,16 +436,14 @@ def run_vanilla_tests(): print('____' + ' '.join(command)) subprocess.check_call(command) - for c in sorted(os.listdir(os.path.join(options.binaryen_test, 'wasm_backend'))): - if not c.endswith('cpp'): - continue - print('..', c) + for c in get_tests(get_test_dir('wasm_backend'), '.cpp'): + print('..', os.path.basename(c)) base = c.replace('.cpp', '').replace('.c', '') - expected = open(os.path.join(options.binaryen_test, 'wasm_backend', base + '.txt')).read() + expected = open(os.path.join(base + '.txt')).read() for opts in [[], ['-O1'], ['-O2']]: # only my code is a hack we used early in wasm backend dev, which somehow worked, but only with -O1 only = [] if opts != ['-O1'] or '_only' not in base else ['-s', 'ONLY_MY_CODE=1'] - command = [VANILLA_EMCC, '-o', 'a.wasm.js', os.path.join(options.binaryen_test, 'wasm_backend', c)] + opts + only + command = [VANILLA_EMCC, '-o', 'a.wasm.js', c] + opts + only print('....' + ' '.join(command)) if os.path.exists('a.wasm.js'): os.unlink('a.wasm.js') @@ -492,12 +467,12 @@ def run_gcc_tests(): fail_with_error('Native compiler (e.g. gcc/g++) was not found in PATH!') return - for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'example'))): + for t in sorted(os.listdir(get_test_dir('example'))): output_file = 'example' cmd = ['-I' + os.path.join(options.binaryen_root, 'src'), '-g', '-pthread', '-o', output_file] if t.endswith('.txt'): # check if there is a trace in the file, if so, we should build it - out = subprocess.check_output([os.path.join(options.binaryen_root, 'scripts', 'clean_c_api_trace.py'), os.path.join(options.binaryen_test, 'example', t)]) + out = subprocess.check_output([os.path.join(options.binaryen_root, 'scripts', 'clean_c_api_trace.py'), os.path.join(get_test_dir('example'), t)]) if len(out) == 0: print(' (no trace in ', t, ')') continue @@ -505,10 +480,10 @@ def run_gcc_tests(): src = 'trace.cpp' with open(src, 'wb') as o: o.write(out) - expected = os.path.join(options.binaryen_test, 'example', t + '.txt') + expected = os.path.join(get_test_dir('example'), t + '.txt') else: - src = os.path.join(options.binaryen_test, 'example', t) - expected = os.path.join(options.binaryen_test, 'example', '.'.join(t.split('.')[:-1]) + '.txt') + src = os.path.join(get_test_dir('example'), t) + expected = os.path.join(get_test_dir('example'), '.'.join(t.split('.')[:-1]) + '.txt') if src.endswith(('.c', '.cpp')): # build the C file separately libpath = os.path.join(os.path.dirname(options.binaryen_bin), 'lib') diff --git a/scripts/test/asm2wasm.py b/scripts/test/asm2wasm.py index 356a3d85c..662daf387 100755 --- a/scripts/test/asm2wasm.py +++ b/scripts/test/asm2wasm.py @@ -20,19 +20,18 @@ import subprocess from .support import run_command from .shared import ( ASM2WASM, WASM_OPT, binary_format_check, delete_from_orbit, - fail_with_error, options, tests, fail_if_not_identical_to_file + fail_with_error, options, fail_if_not_identical_to_file, get_tests ) def test_asm2wasm(): print('[ checking asm2wasm testcases... ]\n') - for asm in tests: - if not asm.endswith('.asm.js'): - continue + for asm in get_tests(options.binaryen_test, ['.asm.js']): + basename = os.path.basename(asm) for precise in [0, 1, 2]: for opts in [1, 0]: - cmd = ASM2WASM + [os.path.join(options.binaryen_test, asm)] + cmd = ASM2WASM + [asm] if 'threads' in asm: cmd += ['--enable-threads'] wasm = asm.replace('.asm.js', '.fromasm') @@ -48,22 +47,21 @@ def test_asm2wasm(): cmd += ['-O0'] # test that -O0 does nothing else: cmd += ['-O'] - if 'debugInfo' in asm: + if 'debugInfo' in basename: cmd += ['-g'] - if 'noffi' in asm: + if 'noffi' in basename: cmd += ['--no-legalize-javascript-ffi'] if precise and opts: # test mem init importing - open('a.mem', 'w').write(asm) + open('a.mem', 'w').write(basename) cmd += ['--mem-init=a.mem'] - if asm[0] == 'e': + if basename[0] == 'e': cmd += ['--mem-base=1024'] - if '4GB' in asm: + if '4GB' in basename: cmd += ['--mem-max=4294967296'] - if 'i64' in asm or 'wasm-only' in asm or 'noffi' in asm: + if 'i64' in basename or 'wasm-only' in basename or 'noffi' in basename: cmd += ['--wasm-only'] - wasm = os.path.join(options.binaryen_test, wasm) - print('..', asm, wasm) + print('..', basename, os.path.basename(wasm)) def do_asm2wasm_test(): actual = run_command(cmd) diff --git a/scripts/test/lld.py b/scripts/test/lld.py index 7372698cd..41db2baa2 100755 --- a/scripts/test/lld.py +++ b/scripts/test/lld.py @@ -16,7 +16,7 @@ import os from .support import run_command from .shared import ( - fail_with_error, files_with_pattern, options, + fail_with_error, get_test_dir, get_tests, WASM_EMSCRIPTEN_FINALIZE, fail_if_not_identical_to_file ) @@ -35,7 +35,7 @@ def args_for_finalize(filename): def test_wasm_emscripten_finalize(): print('\n[ checking wasm-emscripten-finalize testcases... ]\n') - for wast_path in files_with_pattern(options.binaryen_test, 'lld', '*.wast'): + for wast_path in get_tests(get_test_dir('lld'), ['.wast']): print('..', wast_path) is_passive = '.passive.' in wast_path mem_file = wast_path + '.mem' @@ -69,7 +69,7 @@ def test_wasm_emscripten_finalize(): def update_lld_tests(): print('\n[ updatring wasm-emscripten-finalize testcases... ]\n') - for wast_path in files_with_pattern(options.binaryen_test, 'lld', '*.wast'): + for wast_path in get_tests(get_test_dir('lld'), ['.wast']): print('..', wast_path) is_passive = '.passive.' in wast_path mem_file = wast_path + '.mem' diff --git a/scripts/test/shared.py b/scripts/test/shared.py index f8d575656..bcaffe0b8 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -373,10 +373,22 @@ def fail_if_not_identical_to_file(actual, expected_file): fail_if_not_identical(actual, f.read(), fromfile=expected_file) -if len(requested) == 0: - tests = sorted(os.listdir(os.path.join(options.binaryen_test))) -else: - tests = requested[:] +def get_test_dir(name): + """Returns the test directory located at BINARYEN_ROOT/test/[name].""" + return os.path.join(options.binaryen_test, name) + + +def get_tests(test_dir, extensions=[]): + """Returns the list of test files in a given directory. 'extensions' is a + list of file extensions. If 'extensions' is empty, returns all files. + """ + tests = [] + if not extensions: + tests += glob.glob(os.path.join(test_dir, '*')) + for ext in extensions: + tests += glob.glob(os.path.join(test_dir, '*' + ext)) + return sorted(tests) + if not options.interpreter: warn('no interpreter provided (did not test spec interpreter validation)') @@ -457,10 +469,6 @@ def minify_check(wast, verify_final_result=True): os.unlink('b.wast') -def files_with_pattern(*path_pattern): - return sorted(glob.glob(os.path.join(*path_pattern))) - - # run a check with BINARYEN_PASS_DEBUG set, to do full validation def with_pass_debug(check): old_pass_debug = os.environ.get('BINARYEN_PASS_DEBUG') diff --git a/scripts/test/wasm2js.py b/scripts/test/wasm2js.py index 5e30ae0d7..f5bbb4199 100755 --- a/scripts/test/wasm2js.py +++ b/scripts/test/wasm2js.py @@ -19,17 +19,13 @@ import os from .support import run_command, split_wast, write_wast from .shared import ( WASM2JS, MOZJS, NODEJS, fail_if_not_identical, options, - fail_if_not_identical_to_file, with_pass_debug + fail_if_not_identical_to_file, with_pass_debug, get_test_dir, get_tests, ) -tests = sorted(os.listdir(os.path.join(options.binaryen_test))) -spec_dir = os.path.join(options.binaryen_test, 'spec') -spec_tests = [os.path.join(spec_dir, t) - for t in sorted(os.listdir(spec_dir)) - if '.fail' not in t] -wasm2js_dir = os.path.join(options.binaryen_test, 'wasm2js') -extra_wasm2js_tests = [os.path.join(wasm2js_dir, t) for t in - sorted(os.listdir(wasm2js_dir))] +tests = get_tests(options.binaryen_test) +spec_tests = get_tests(get_test_dir('spec'), ['.wast']) +spec_tests = [t for t in spec_tests if '.fail' not in t] +wasm2js_tests = get_tests(get_test_dir('wasm2js'), ['.wast']) assert_tests = ['wasm2js.wast.asserts'] # These tests exercise functionality not supported by wasm2js wasm2js_blacklist = ['empty_imported_table.wast'] @@ -37,24 +33,20 @@ wasm2js_blacklist = ['empty_imported_table.wast'] def test_wasm2js_output(): for opt in (0, 1): - for wasm in tests + spec_tests + extra_wasm2js_tests: - if not wasm.endswith('.wast'): - continue - basename = os.path.basename(wasm) + for t in tests + spec_tests + wasm2js_tests: + basename = os.path.basename(t) if basename in wasm2js_blacklist: continue asm = basename.replace('.wast', '.2asm.js') - expected_file = os.path.join(wasm2js_dir, asm) + expected_file = os.path.join(get_test_dir('wasm2js'), asm) if opt: expected_file += '.opt' if not os.path.exists(expected_file): continue - print('..', wasm) - - t = os.path.join(options.binaryen_test, wasm) + print('..', os.path.basename(t)) all_out = [] @@ -64,7 +56,7 @@ def test_wasm2js_output(): cmd = WASM2JS + ['split.wast', '-all'] if opt: cmd += ['-O'] - if 'emscripten' in wasm: + if 'emscripten' in t: cmd += ['--emscripten'] out = run_command(cmd) all_out.append(out) @@ -109,7 +101,7 @@ def test_asserts_output(): asserts_expected_file = os.path.join(options.binaryen_test, asserts) traps_expected_file = os.path.join(options.binaryen_test, traps) - wasm = os.path.join(wasm2js_dir, wasm) + wasm = os.path.join(get_test_dir('wasm2js'), wasm) cmd = WASM2JS + [wasm, '--allow-asserts', '-all'] out = run_command(cmd) fail_if_not_identical_to_file(out, asserts_expected_file) @@ -129,7 +121,7 @@ def update_wasm2js_tests(): print('\n[ checking wasm2js ]\n') for opt in (0, 1): - for wasm in tests + spec_tests + extra_wasm2js_tests: + for wasm in tests + spec_tests + wasm2js_tests: if not wasm.endswith('.wast'): continue @@ -137,7 +129,7 @@ def update_wasm2js_tests(): continue asm = os.path.basename(wasm).replace('.wast', '.2asm.js') - expected_file = os.path.join(wasm2js_dir, asm) + expected_file = os.path.join(get_test_dir('wasm2js', asm)) if opt: expected_file += '.opt' @@ -145,7 +137,7 @@ def update_wasm2js_tests(): # exists - only some work so far. the tests in extra are in # the test/wasm2js dir and so are specific to wasm2js, and # we run all of those. - if wasm not in extra_wasm2js_tests and not os.path.exists(expected_file): + if wasm not in wasm2js_tests and not os.path.exists(expected_file): continue print('..', wasm) @@ -176,7 +168,7 @@ def update_wasm2js_tests(): asserts_expected_file = os.path.join(options.binaryen_test, asserts) traps_expected_file = os.path.join(options.binaryen_test, traps) - cmd = WASM2JS + [os.path.join(wasm2js_dir, wasm), '--allow-asserts', '-all'] + cmd = WASM2JS + [os.path.join(get_test_dir('wasm2js'), wasm), '--allow-asserts', '-all'] out = run_command(cmd) with open(asserts_expected_file, 'w') as o: o.write(out) |