diff options
| -rw-r--r-- | .flake8 | 3 | ||||
| -rw-r--r-- | .travis.yml | 3 | ||||
| -rwxr-xr-x | auto_update_tests.py | 717 | ||||
| -rwxr-xr-x | check.py | 123 |
4 files changed, 484 insertions, 362 deletions
diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..c1d8f5c58 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +ignore = E111,E114,E501,E121 +exclude = ./test/ diff --git a/.travis.yml b/.travis.yml index a41368a67..6a01f25f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,8 +30,7 @@ jobs: - export PATH="${HOME}/.jsvu:${PATH}" - jsvu --os=linux64 --engines=spidermonkey before_script: - # Check the style of a subset of Python code until the other code is updated. - - flake8 ./scripts/ + - flake8 - ./check.py --test-waterfall --only-prepare script: - cmake . -DCMAKE_C_FLAGS="$COMPILER_FLAGS" -DCMAKE_CXX_FLAGS="$COMPILER_FLAGS" diff --git a/auto_update_tests.py b/auto_update_tests.py index 73245d524..50e9c73b9 100755 --- a/auto_update_tests.py +++ b/auto_update_tests.py @@ -1,266 +1,319 @@ #!/usr/bin/env python - -import os, sys, subprocess, difflib +# Copyright 2015 WebAssembly Community Group participants +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import subprocess +import sys from scripts.test.support import run_command, split_wast, node_test_glue, node_has_webassembly from scripts.test.shared import ( - ASM2WASM, MOZJS, NODEJS, S2WASM, WASM_SHELL, WASM_OPT, WASM_AS, WASM_DIS, + ASM2WASM, MOZJS, NODEJS, S2WASM, WASM_OPT, WASM_AS, WASM_DIS, WASM_CTOR_EVAL, WASM_MERGE, WASM_REDUCE, WASM2ASM, WASM_METADCE, WASM_EMSCRIPTEN_FINALIZE, BINARYEN_INSTALL_DIR, files_with_pattern, has_shell_timeout) from scripts.test.wasm2asm import tests, spec_tests, extra_wasm2asm_tests, assert_tests, wasm2asm_dir -print '[ processing and updating testcases... ]\n' - -for asm in sorted(os.listdir('test')): - if asm.endswith('.asm.js'): - for precise in [0, 1, 2]: - for opts in [1, 0]: - cmd = ASM2WASM + [os.path.join('test', asm), '--enable-threads'] - wasm = asm.replace('.asm.js', '.fromasm') - if not precise: - cmd += ['--trap-mode=allow', '--ignore-implicit-traps'] - wasm += '.imprecise' - elif precise == 2: - cmd += ['--trap-mode=clamp'] - wasm += '.clamp' - if not opts: - wasm += '.no-opts' - if precise: - cmd += ['-O0'] # test that -O0 does nothing - else: - cmd += ['-O'] - if 'debugInfo' in asm: + +def update_asm_js_tests(): + print '[ processing and updating testcases... ]\n' + for asm in sorted(os.listdir('test')): + if asm.endswith('.asm.js'): + for precise in [0, 1, 2]: + for opts in [1, 0]: + cmd = ASM2WASM + [os.path.join('test', asm), '--enable-threads'] + wasm = asm.replace('.asm.js', '.fromasm') + if not precise: + cmd += ['--trap-mode=allow', '--ignore-implicit-traps'] + wasm += '.imprecise' + elif precise == 2: + cmd += ['--trap-mode=clamp'] + wasm += '.clamp' + if not opts: + wasm += '.no-opts' + if precise: + cmd += ['-O0'] # test that -O0 does nothing + else: + cmd += ['-O'] + if 'debugInfo' in asm: + cmd += ['-g'] + if 'noffi' in asm: + cmd += ['--no-legalize-javascript-ffi'] + if precise and opts: + # test mem init importing + open('a.mem', 'wb').write(asm) + cmd += ['--mem-init=a.mem'] + if asm[0] == 'e': + cmd += ['--mem-base=1024'] + if 'i64' in asm or 'wasm-only' in asm or 'noffi' in asm: + cmd += ['--wasm-only'] + print ' '.join(cmd) + actual = run_command(cmd) + with open(os.path.join('test', wasm), 'w') as o: + o.write(actual) + if 'debugInfo' in asm: + cmd += ['--source-map', os.path.join('test', wasm + '.map'), '-o', 'a.wasm'] + run_command(cmd) + + +def update_dot_s_tests(): + extension_arg_map = { + '.wast': [], + '.clamp.wast': ['--trap-mode=clamp'], + '.js.wast': ['--trap-mode=js'], + '.jscall.wast': ['--emscripten-reserved-function-pointers=3'], + } + for dot_s_dir in ['dot_s', 'llvm_autogenerated']: + for s in sorted(os.listdir(os.path.join('test', dot_s_dir))): + if not s.endswith('.s'): + continue + print '..', s + for ext, ext_args in extension_arg_map.iteritems(): + wasm = s.replace('.s', ext) + expected_file = os.path.join('test', dot_s_dir, wasm) + if ext != '.wast' and not os.path.exists(expected_file): + continue + + full = os.path.join('test', dot_s_dir, s) + stack_alloc = ['--allocate-stack=1024'] if dot_s_dir == 'llvm_autogenerated' else [] + cmd = S2WASM + [full, '--emscripten-glue'] + stack_alloc + ext_args + if s.startswith('start_'): + cmd.append('--start') + actual = run_command(cmd, stderr=subprocess.PIPE, expected_err='') + + with open(expected_file, 'w') as o: + o.write(actual) + + +def update_lld_tests(): + print '\n[ checking wasm-emscripten-finalize testcases... ]\n' + + extension_arg_map = { + '.out': [], + '.jscall.out': ['--emscripten-reserved-function-pointers=3'], + } + for wast_path in files_with_pattern('test', 'lld', '*.wast'): + print '..', wast_path + for ext, ext_args in extension_arg_map.items(): + out_path = wast_path + ext + if ext != '.out' and not os.path.exists(out_path): + continue + cmd = (WASM_EMSCRIPTEN_FINALIZE + + [wast_path, '-S', '--global-base=568'] + ext_args) + actual = run_command(cmd) + with open(out_path, 'w') as o: + o.write(actual) + + +def update_wasm_opt_tests(): + print '\n[ checking wasm-opt -o notation... ]\n' + wast = os.path.join('test', 'hello_world.wast') + cmd = WASM_OPT + [wast, '-o', 'a.wast', '-S'] + run_command(cmd) + open(wast, 'w').write(open('a.wast').read()) + + print '\n[ checking wasm-opt binary reading/writing... ]\n' + for t in sorted(os.listdir(os.path.join('test', 'print'))): + if t.endswith('.wast'): + print '..', t + wasm = os.path.basename(t).replace('.wast', '') + cmd = WASM_OPT + [os.path.join('test', 'print', t), '--print'] + print ' ', ' '.join(cmd) + actual = subprocess.check_output(cmd) + print cmd, actual + with open(os.path.join('test', 'print', wasm + '.txt'), 'w') as o: + o.write(actual) + cmd = WASM_OPT + [os.path.join('test', 'print', t), '--print-minified'] + print ' ', ' '.join(cmd) + actual = subprocess.check_output(cmd) + with open(os.path.join('test', 'print', wasm + '.minified.txt'), 'w') as o: + o.write(actual) + + print '\n[ checking wasm-opt passes... ]\n' + for t in sorted(os.listdir(os.path.join('test', 'passes'))): + if t.endswith(('.wast', '.wasm')): + print '..', t + binary = '.wasm' in t + base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') + passname = base + if passname.isdigit(): + passname = open(os.path.join('test', 'passes', passname + '.passes')).read().strip() + opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] + t = os.path.join('test', 'passes', t) + actual = '' + for module, asserts in split_wast(t): + assert len(asserts) == 0 + with open('split.wast', 'w') as o: + o.write(module) + cmd = WASM_OPT + opts + ['split.wast', '--print'] + actual += run_command(cmd) + with open(os.path.join('test', 'passes', base + ('.bin' if binary else '') + '.txt'), 'w') as o: + o.write(actual) + if 'emit-js-wrapper' in t: + with open('a.js') as i: + with open(t + '.js', 'w') as o: + o.write(i.read()) + if 'emit-spec-wrapper' in t: + with open('a.wat') as i: + with open(t + '.wat', 'w') as o: + o.write(i.read()) + + print '\n[ checking wasm-opt testcases... ]\n' + for t in os.listdir('test'): + if t.endswith('.wast') and not t.startswith('spec'): + print '..', t + t = os.path.join('test', t) + f = t + '.from-wast' + cmd = WASM_OPT + [t, '--print'] + actual = run_command(cmd) + actual = actual.replace('printing before:\n', '') + open(f, 'w').write(actual) + + +def update_bin_fmt_tests(): + print '\n[ checking binary format testcases... ]\n' + for wast in sorted(os.listdir('test')): + if wast.endswith('.wast') and wast not in []: # blacklist some known failures + for debug_info in [0, 1]: + cmd = WASM_AS + [os.path.join('test', wast), '-o', 'a.wasm'] + if debug_info: cmd += ['-g'] - if 'noffi' in asm: - cmd += ['--no-legalize-javascript-ffi'] - if precise and opts: - # test mem init importing - open('a.mem', 'wb').write(asm) - cmd += ['--mem-init=a.mem'] - if asm[0] == 'e': - cmd += ['--mem-base=1024'] - if 'i64' in asm or 'wasm-only' in asm or 'noffi' in asm: - cmd += ['--wasm-only'] print ' '.join(cmd) - actual = run_command(cmd) - with open(os.path.join('test', wasm), 'w') as o: o.write(actual) - if 'debugInfo' in asm: - cmd += ['--source-map', os.path.join('test', wasm + '.map'), '-o', 'a.wasm'] - run_command(cmd) - -extension_arg_map = { - '.wast': [], - '.clamp.wast': ['--trap-mode=clamp'], - '.js.wast': ['--trap-mode=js'], - '.jscall.wast': ['--emscripten-reserved-function-pointers=3'], -} -for dot_s_dir in ['dot_s', 'llvm_autogenerated']: - for s in sorted(os.listdir(os.path.join('test', dot_s_dir))): - if not s.endswith('.s'): continue - print '..', s - for ext, ext_args in extension_arg_map.iteritems(): - wasm = s.replace('.s', ext) - expected_file = os.path.join('test', dot_s_dir, wasm) - if ext != '.wast' and not os.path.exists(expected_file): - continue + if os.path.exists('a.wasm'): + os.unlink('a.wasm') + subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert os.path.exists('a.wasm') - full = os.path.join('test', dot_s_dir, s) - stack_alloc = ['--allocate-stack=1024'] if dot_s_dir == 'llvm_autogenerated' else [] - cmd = S2WASM + [full, '--emscripten-glue'] + stack_alloc + ext_args - if s.startswith('start_'): - cmd.append('--start') - actual = run_command(cmd, stderr=subprocess.PIPE, expected_err='') - - with open(expected_file, 'w') as o: o.write(actual) - -print '\n[ checking wasm-emscripten-finalize testcases... ]\n' - -emscripten_finalize_extension_arg_map = { - '.out': [], - '.jscall.out': ['--emscripten-reserved-function-pointers=3'], -} -for wast_path in files_with_pattern('test', 'lld', '*.wast'): - print '..', wast_path - for ext, ext_args in emscripten_finalize_extension_arg_map.items(): - out_path = wast_path + ext - if ext != '.out' and not os.path.exists(out_path): - continue - cmd = (WASM_EMSCRIPTEN_FINALIZE + - [wast_path, '-S', '--global-base=568'] + ext_args) - actual = run_command(cmd) - with open(out_path, 'w') as o: o.write(actual) - -for t in sorted(os.listdir(os.path.join('test', 'print'))): - if t.endswith('.wast'): - print '..', t - wasm = os.path.basename(t).replace('.wast', '') - cmd = WASM_OPT + [os.path.join('test', 'print', t), '--print'] - print ' ', ' '.join(cmd) - actual = subprocess.check_output(cmd) - print cmd, actual - with open(os.path.join('test', 'print', wasm + '.txt'), 'w') as o: o.write(actual) - cmd = WASM_OPT + [os.path.join('test', 'print', t), '--print-minified'] - print ' ', ' '.join(cmd) - actual = subprocess.check_output(cmd) - with open(os.path.join('test', 'print', wasm + '.minified.txt'), 'w') as o: o.write(actual) - -for t in sorted(os.listdir(os.path.join('test', 'passes'))): - if t.endswith(('.wast', '.wasm')): - print '..', t - binary = '.wasm' in t - base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') - passname = base - if passname.isdigit(): - passname = open(os.path.join('test', 'passes', passname + '.passes')).read().strip() - opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] - t = os.path.join('test', 'passes', t) - actual = '' - for module, asserts in split_wast(t): - assert len(asserts) == 0 - with open('split.wast', 'w') as o: o.write(module) - cmd = WASM_OPT + opts + ['split.wast', '--print'] - actual += run_command(cmd) - with open(os.path.join('test', 'passes', base + ('.bin' if binary else '') + '.txt'), 'w') as o: o.write(actual) - if 'emit-js-wrapper' in t: - with open('a.js') as i: - with open(t + '.js', 'w') as o: - o.write(i.read()) - if 'emit-spec-wrapper' in t: - with open('a.wat') as i: - with open(t + '.wat', 'w') as o: - o.write(i.read()) - -print '\n[ checking wasm-opt -o notation... ]\n' - -wast = os.path.join('test', 'hello_world.wast') -cmd = WASM_OPT + [wast, '-o', 'a.wast', '-S'] -run_command(cmd) -open(wast, 'w').write(open('a.wast').read()) - -print '\n[ checking binary format testcases... ]\n' - -for wast in sorted(os.listdir('test')): - if wast.endswith('.wast') and not wast in []: # blacklist some known failures - for debug_info in [0, 1]: - cmd = WASM_AS + [os.path.join('test', wast), '-o', 'a.wasm'] - if debug_info: cmd += ['-g'] - print ' '.join(cmd) - if os.path.exists('a.wasm'): os.unlink('a.wasm') - subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert os.path.exists('a.wasm') - - cmd = WASM_DIS + ['a.wasm', '-o', 'a.wast'] - print ' '.join(cmd) - if os.path.exists('a.wast'): os.unlink('a.wast') - subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert os.path.exists('a.wast') - actual = open('a.wast').read() - binary_name = wast + '.fromBinary' - if not debug_info: binary_name += '.noDebugInfo' - with open(os.path.join('test', binary_name), 'w') as o: o.write(actual) - -print '\n[ checking example testcases... ]\n' - -for t in sorted(os.listdir(os.path.join('test', 'example'))): - output_file = os.path.join('bin', 'example') - libdir = os.path.join(BINARYEN_INSTALL_DIR, 'lib') - cmd = ['-Isrc', '-g', '-pthread', '-o', output_file] - if t.endswith('.txt'): - # check if there is a trace in the file, if so, we should build it - out = subprocess.Popen([os.path.join('scripts', 'clean_c_api_trace.py'), os.path.join('test', 'example', t)], stdout=subprocess.PIPE).communicate()[0] - if len(out) == 0: - print ' (no trace in ', t, ')' - continue - print ' (will check trace in ', t, ')' - src = 'trace.cpp' - with open(src, 'w') as o: o.write(out) - expected = os.path.join('test', 'example', t + '.txt') - else: - src = os.path.join('test', 'example', t) - expected = os.path.join('test', 'example', '.'.join(t.split('.')[:-1]) + '.txt') - if not src.endswith(('.c', '.cpp')): - continue - # build the C file separately - extra = [os.environ.get('CC') or 'gcc', - src, '-c', '-o', 'example.o', - '-Isrc', '-g', '-L' + libdir, '-pthread'] - print 'build: ', ' '.join(extra) - if src.endswith('.cpp'): - extra += ['-std=c++11'] - print os.getcwd() - subprocess.check_call(extra) - # Link against the binaryen C library DSO, using rpath - cmd = ['example.o', '-L' + libdir, '-lbinaryen', '-Wl,-rpath=' + os.path.abspath(libdir)] + cmd - print ' ', t, src, expected - if os.environ.get('COMPILER_FLAGS'): - for f in os.environ.get('COMPILER_FLAGS').split(' '): - cmd.append(f) - cmd = [os.environ.get('CXX') or 'g++', '-std=c++11'] + cmd - try: - print 'link: ', ' '.join(cmd) - subprocess.check_call(cmd) - print 'run...', output_file - proc = subprocess.Popen([output_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - actual, err = proc.communicate() - assert proc.returncode == 0, [proc.returncode, actual, err] - with open(expected, 'w') as o: o.write(actual) - finally: - os.remove(output_file) - if sys.platform == 'darwin': - # Also removes debug directory produced on Mac OS - shutil.rmtree(output_file + '.dSYM') - -print '\n[ checking wasm-opt testcases... ]\n' - -for t in os.listdir('test'): - if t.endswith('.wast') and not t.startswith('spec'): - print '..', t - t = os.path.join('test', t) - f = t + '.from-wast' - cmd = WASM_OPT + [t, '--print'] - actual = run_command(cmd) - actual = actual.replace('printing before:\n', '') - open(f, 'w').write(actual) - -print '\n[ checking wasm-dis on provided binaries... ]\n' - -for t in os.listdir('test'): - if t.endswith('.wasm') and not t.startswith('spec'): - print '..', t - t = os.path.join('test', t) - cmd = WASM_DIS + [t] - if os.path.isfile(t + '.map'): cmd += ['--source-map', t + '.map'] - actual = run_command(cmd) - - open(t + '.fromBinary', 'w').write(actual) - -print '\n[ checking wasm-merge... ]\n' - -for t in os.listdir(os.path.join('test', 'merge')): - if t.endswith(('.wast', '.wasm')): - print '..', t - t = os.path.join('test', 'merge', t) - u = t + '.toMerge' - for finalize in [0, 1]: - for opt in [0, 1]: - cmd = WASM_MERGE + [t, u, '-o', 'a.wast', '-S', '--verbose'] - if finalize: cmd += ['--finalize-memory-base=1024', '--finalize-table-base=8'] - if opt: cmd += ['-O'] - stdout = run_command(cmd) + cmd = WASM_DIS + ['a.wasm', '-o', 'a.wast'] + print ' '.join(cmd) + if os.path.exists('a.wast'): + os.unlink('a.wast') + subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert os.path.exists('a.wast') actual = open('a.wast').read() - out = t + '.combined' - if finalize: out += '.finalized' - if opt: out += '.opt' - with open(out, 'w') as o: o.write(actual) - with open(out + '.stdout', 'w') as o: o.write(stdout) + binary_name = wast + '.fromBinary' + if not debug_info: + binary_name += '.noDebugInfo' + with open(os.path.join('test', binary_name), 'w') as o: + o.write(actual) + + +def update_example_tests(): + print '\n[ checking example testcases... ]\n' + for t in sorted(os.listdir(os.path.join('test', 'example'))): + output_file = os.path.join('bin', 'example') + libdir = os.path.join(BINARYEN_INSTALL_DIR, 'lib') + cmd = ['-Isrc', '-g', '-pthread', '-o', output_file] + if t.endswith('.txt'): + # check if there is a trace in the file, if so, we should build it + out = subprocess.Popen([os.path.join('scripts', 'clean_c_api_trace.py'), os.path.join('test', 'example', t)], stdout=subprocess.PIPE).communicate()[0] + if len(out) == 0: + print ' (no trace in ', t, ')' + continue + print ' (will check trace in ', t, ')' + src = 'trace.cpp' + with open(src, 'w') as o: + o.write(out) + expected = os.path.join('test', 'example', t + '.txt') + else: + src = os.path.join('test', 'example', t) + expected = os.path.join('test', 'example', '.'.join(t.split('.')[:-1]) + '.txt') + if not src.endswith(('.c', '.cpp')): + continue + # build the C file separately + extra = [os.environ.get('CC') or 'gcc', + src, '-c', '-o', 'example.o', + '-Isrc', '-g', '-L' + libdir, '-pthread'] + print 'build: ', ' '.join(extra) + if src.endswith('.cpp'): + extra += ['-std=c++11'] + print os.getcwd() + subprocess.check_call(extra) + # Link against the binaryen C library DSO, using rpath + cmd = ['example.o', '-L' + libdir, '-lbinaryen', '-Wl,-rpath=' + os.path.abspath(libdir)] + cmd + print ' ', t, src, expected + if os.environ.get('COMPILER_FLAGS'): + for f in os.environ.get('COMPILER_FLAGS').split(' '): + cmd.append(f) + cmd = [os.environ.get('CXX') or 'g++', '-std=c++11'] + cmd + try: + print 'link: ', ' '.join(cmd) + subprocess.check_call(cmd) + print 'run...', output_file + proc = subprocess.Popen([output_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + actual, err = proc.communicate() + assert proc.returncode == 0, [proc.returncode, actual, err] + with open(expected, 'w') as o: + o.write(actual) + finally: + os.remove(output_file) + if sys.platform == 'darwin': + # Also removes debug directory produced on Mac OS + shutil.rmtree(output_file + '.dSYM') + + +def update_wasm_dis_tests(): + print '\n[ checking wasm-dis on provided binaries... ]\n' + for t in os.listdir('test'): + if t.endswith('.wasm') and not t.startswith('spec'): + print '..', t + t = os.path.join('test', t) + cmd = WASM_DIS + [t] + if os.path.isfile(t + '.map'): + cmd += ['--source-map', t + '.map'] + actual = run_command(cmd) + + open(t + '.fromBinary', 'w').write(actual) -if MOZJS or NODEJS: - print '\n[ checking binaryen.js testcases... ]\n' +def update_wasm_merge_tests(): + print '\n[ checking wasm-merge... ]\n' + for t in os.listdir(os.path.join('test', 'merge')): + if t.endswith(('.wast', '.wasm')): + print '..', t + t = os.path.join('test', 'merge', t) + u = t + '.toMerge' + for finalize in [0, 1]: + for opt in [0, 1]: + cmd = WASM_MERGE + [t, u, '-o', 'a.wast', '-S', '--verbose'] + if finalize: + cmd += ['--finalize-memory-base=1024', '--finalize-table-base=8'] + if opt: + cmd += ['-O'] + stdout = run_command(cmd) + actual = open('a.wast').read() + out = t + '.combined' + if finalize: + out += '.finalized' + if opt: + out += '.opt' + with open(out, 'w') as o: + o.write(actual) + with open(out + '.stdout', 'w') as o: + o.write(stdout) + + +def update_binaryen_js_tests(): + if not (MOZJS or NODEJS): + return + + print '\n[ checking binaryen.js testcases... ]\n' node_has_wasm = NODEJS and node_has_webassembly(NODEJS) for s in sorted(os.listdir(os.path.join('test', 'binaryen.js'))): - if not s.endswith('.js'): continue + if not s.endswith('.js'): + continue print s f = open('a.js', 'w') f.write(open(os.path.join('bin', 'binaryen.js')).read()) @@ -270,81 +323,93 @@ if MOZJS or NODEJS: test_src = open(test_path).read() f.write(test_src) f.close() - if MOZJS or node_has_wasm or not 'WebAssembly.' in test_src: + if MOZJS or node_has_wasm or 'WebAssembly.' not in test_src: cmd = [MOZJS or NODEJS, 'a.js'] out = run_command(cmd, stderr=subprocess.STDOUT) - with open(os.path.join('test', 'binaryen.js', s + '.txt'), 'w') as o: o.write(out) + with open(os.path.join('test', 'binaryen.js', s + '.txt'), 'w') as o: + o.write(out) else: print 'Skipping ' + test_path + ' because WebAssembly might not be supported' -print '\n[ checking wasm-ctor-eval... ]\n' - -for t in os.listdir(os.path.join('test', 'ctor-eval')): - if t.endswith(('.wast', '.wasm')): - print '..', t - t = os.path.join('test', 'ctor-eval', t) - ctors = open(t + '.ctors').read().strip() - cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] - stdout = run_command(cmd) - actual = open('a.wast').read() - out = t + '.out' - with open(out, 'w') as o: o.write(actual) - -print '\n[ checking wasm2asm ]\n' - -for wasm in tests + spec_tests + extra_wasm2asm_tests: - if not wasm.endswith('.wast'): - continue - - asm = os.path.basename(wasm).replace('.wast', '.2asm.js') - expected_file = os.path.join(wasm2asm_dir, asm) - - # we run wasm2asm on tests and spec tests only if the output - # exists - only some work so far. the tests in extra are in - # the test/wasm2asm dir and so are specific to wasm2asm, and - # we run all of those. - if wasm not in extra_wasm2asm_tests and not os.path.exists(expected_file): - continue - - print '..', wasm - - cmd = WASM2ASM + [os.path.join('test', wasm)] - out = run_command(cmd) - with open(expected_file, 'w') as o: o.write(out) - -for wasm in assert_tests: - print '..', wasm - - asserts = os.path.basename(wasm).replace('.wast.asserts', '.asserts.js') - traps = os.path.basename(wasm).replace('.wast.asserts', '.traps.js') - asserts_expected_file = os.path.join('test', asserts) - traps_expected_file = os.path.join('test', traps) - - cmd = WASM2ASM + [os.path.join(wasm2asm_dir, wasm), '--allow-asserts'] - out = run_command(cmd) - with open(asserts_expected_file, 'w') as o: o.write(out) - - cmd += ['--pedantic'] - out = run_command(cmd) - with open(traps_expected_file, 'w') as o: o.write(out) - -print '\n[ checking wasm-metadce... ]\n' - -for t in os.listdir(os.path.join('test', 'metadce')): - if t.endswith(('.wast', '.wasm')): - print '..', t - t = os.path.join('test', 'metadce', t) - graph = t + '.graph.txt' - cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S'] - stdout = run_command(cmd) - actual = open('a.wast').read() - out = t + '.dced' - with open(out, 'w') as o: o.write(actual) - with open(out + '.stdout', 'w') as o: o.write(stdout) - -if has_shell_timeout(): - print '\n[ checking wasm-reduce ]\n' +def update_ctor_eval_tests(): + print '\n[ checking wasm-ctor-eval... ]\n' + for t in os.listdir(os.path.join('test', 'ctor-eval')): + if t.endswith(('.wast', '.wasm')): + print '..', t + t = os.path.join('test', 'ctor-eval', t) + ctors = open(t + '.ctors').read().strip() + cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] + run_command(cmd) + actual = open('a.wast').read() + out = t + '.out' + with open(out, 'w') as o: + o.write(actual) + + +def update_wasm2asm_tests(): + print '\n[ checking wasm2asm ]\n' + for wasm in tests + spec_tests + extra_wasm2asm_tests: + if not wasm.endswith('.wast'): + continue + + asm = os.path.basename(wasm).replace('.wast', '.2asm.js') + expected_file = os.path.join(wasm2asm_dir, asm) + + # we run wasm2asm on tests and spec tests only if the output + # exists - only some work so far. the tests in extra are in + # the test/wasm2asm dir and so are specific to wasm2asm, and + # we run all of those. + if wasm not in extra_wasm2asm_tests and not os.path.exists(expected_file): + continue + + print '..', wasm + + cmd = WASM2ASM + [os.path.join('test', wasm)] + out = run_command(cmd) + with open(expected_file, 'w') as o: + o.write(out) + + for wasm in assert_tests: + print '..', wasm + + asserts = os.path.basename(wasm).replace('.wast.asserts', '.asserts.js') + traps = os.path.basename(wasm).replace('.wast.asserts', '.traps.js') + asserts_expected_file = os.path.join('test', asserts) + traps_expected_file = os.path.join('test', traps) + + cmd = WASM2ASM + [os.path.join(wasm2asm_dir, wasm), '--allow-asserts'] + out = run_command(cmd) + with open(asserts_expected_file, 'w') as o: + o.write(out) + + cmd += ['--pedantic'] + out = run_command(cmd) + with open(traps_expected_file, 'w') as o: + o.write(out) + + +def update_metadce_tests(): + print '\n[ checking wasm-metadce... ]\n' + for t in os.listdir(os.path.join('test', 'metadce')): + if t.endswith(('.wast', '.wasm')): + print '..', t + t = os.path.join('test', 'metadce', t) + graph = t + '.graph.txt' + cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S'] + stdout = run_command(cmd) + actual = open('a.wast').read() + out = t + '.dced' + with open(out, 'w') as o: + o.write(actual) + with open(out + '.stdout', 'w') as o: + o.write(stdout) + + +def update_reduce_tests(): + if not has_shell_timeout(): + return + print '\n[ checking wasm-reduce ]\n' for t in os.listdir(os.path.join('test', 'reduce')): if t.endswith('.wast'): print '..', t @@ -355,4 +420,24 @@ if has_shell_timeout(): expected = t + '.txt' run_command(WASM_DIS + ['c.wasm', '-o', expected]) -print '\n[ success! ]' + +def main(): + update_asm_js_tests() + update_dot_s_tests() + update_lld_tests() + update_wasm_opt_tests() + update_bin_fmt_tests() + update_example_tests() + update_wasm_dis_tests() + update_wasm_merge_tests() + update_binaryen_js_tests() + update_ctor_eval_tests() + update_wasm2asm_tests() + update_metadce_tests() + update_reduce_tests() + + print '\n[ success! ]' + + +if __name__ == '__main__': + sys.exit(main()) @@ -1,5 +1,4 @@ #!/usr/bin/env python2 - # Copyright 2015 WebAssembly Community Group participants # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os import shutil import subprocess @@ -39,6 +37,7 @@ if options.interpreter: print '[ using wasm interpreter at "%s" ]' % options.interpreter assert os.path.exists(options.interpreter), 'interpreter not found' + # run a check with BINARYEN_PASS_DEBUG set, to do full validation def with_pass_debug(check): old_pass_debug = os.environ.get('BINARYEN_PASS_DEBUG') @@ -52,7 +51,6 @@ def with_pass_debug(check): if 'BINARYEN_PASS_DEBUG' in os.environ: del os.environ['BINARYEN_PASS_DEBUG'] -# tests def run_help_tests(): print '[ checking --help is useful... ]\n' @@ -70,6 +68,7 @@ def run_help_tests(): assert e.replace('.exe', '') in err, 'Expected help to contain program name, got:\n%s' % err assert len(err.split('\n')) > 8, 'Expected some help, got:\n%s' % err + def run_wasm_opt_tests(): print '\n[ checking wasm-opt -o notation... ]\n' @@ -105,13 +104,15 @@ def run_wasm_opt_tests(): actual = '' for module, asserts in split_wast(t): assert len(asserts) == 0 - with open('split.wast', 'w') as o: o.write(module) + with open('split.wast', 'w') as o: + o.write(module) cmd = WASM_OPT + opts + ['split.wast', '--print'] curr = run_command(cmd) actual += curr # also check debug mode output is valid debugged = run_command(cmd + ['--debug'], stderr=subprocess.PIPE) fail_if_not_contained(actual, debugged) + # also check pass-debug mode def check(): pass_debug = run_command(cmd) @@ -158,11 +159,12 @@ def run_wasm_opt_tests(): fail_if_not_identical_to_file(actual, f) - binary_format_check(t, wasm_as_args=['-g']) # test with debuginfo - binary_format_check(t, wasm_as_args=[], binary_suffix='.fromBinary.noDebugInfo') # test without debuginfo + binary_format_check(t, wasm_as_args=['-g']) # test with debuginfo + binary_format_check(t, wasm_as_args=[], binary_suffix='.fromBinary.noDebugInfo') # test without debuginfo minify_check(t) + def run_wasm_dis_tests(): print '\n[ checking wasm-dis on provided binaries... ]\n' @@ -171,7 +173,8 @@ def run_wasm_dis_tests(): print '..', t t = os.path.join(options.binaryen_test, t) cmd = WASM_DIS + [t] - if os.path.isfile(t + '.map'): cmd += ['--source-map', t + '.map'] + if os.path.isfile(t + '.map'): + cmd += ['--source-map', t + '.map'] actual = run_command(cmd) fail_if_not_identical_to_file(actual, t + '.fromBinary') @@ -179,9 +182,11 @@ def run_wasm_dis_tests(): # also verify there are no validation errors def check(): cmd = WASM_OPT + [t] - actual = run_command(cmd) + run_command(cmd) + with_pass_debug(check) + def run_wasm_merge_tests(): print '\n[ checking wasm-merge... ]\n' @@ -194,16 +199,21 @@ def run_wasm_merge_tests(): for finalize in [0, 1]: for opt in [0, 1]: cmd = WASM_MERGE + [t, u, '-o', 'a.wast', '-S', '--verbose'] - if finalize: cmd += ['--finalize-memory-base=1024', '--finalize-table-base=8'] - if opt: cmd += ['-O'] + if finalize: + cmd += ['--finalize-memory-base=1024', '--finalize-table-base=8'] + if opt: + cmd += ['-O'] stdout = run_command(cmd) actual = open('a.wast').read() out = t + '.combined' - if finalize: out += '.finalized' - if opt: out += '.opt' + if finalize: + out += '.finalized' + if opt: + out += '.opt' fail_if_not_identical_to_file(actual, out) fail_if_not_identical_to_file(stdout, out + '.stdout') + def run_crash_tests(): print "\n[ checking we don't crash on tricky inputs... ]\n" @@ -216,6 +226,7 @@ def run_crash_tests(): # expect a parse error to be reported run_command(cmd, expected_err='parse exception:', err_contains=True, expected_status=1) + def run_ctor_eval_tests(): print '\n[ checking wasm-ctor-eval... ]\n' @@ -226,11 +237,12 @@ def run_ctor_eval_tests(): t = os.path.join(test_dir, t) ctors = open(t + '.ctors').read().strip() cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors] - stdout = run_command(cmd) + run_command(cmd) actual = open('a.wast').read() out = t + '.out' fail_if_not_identical_to_file(actual, out) + def run_wasm_metadce_tests(): print '\n[ checking wasm-metadce ]\n' @@ -247,6 +259,7 @@ def run_wasm_metadce_tests(): fail_if_not_identical_to_file(seen.read(), expected) fail_if_not_identical_to_file(stdout, expected + '.stdout') + def run_wasm_reduce_tests(): print '\n[ checking wasm-reduce testcases]\n' @@ -275,11 +288,13 @@ def run_wasm_reduce_tests(): after = os.stat('c.wasm').st_size assert after < 0.333 * before, [before, after] + def run_spec_tests(): print '\n[ checking wasm-shell spec testcases... ]\n' if len(requested) == 0: - BLACKLIST = ['memory.wast', 'binary.wast'] # FIXME we support old and new memory formats, for now, until 0xc, and so can't pass this old-style test. + # FIXME we support old and new memory formats, for now, until 0xc, and so can't pass this old-style test. + BLACKLIST = ['memory.wast', 'binary.wast'] # FIXME to update the spec to 0xd, we need to implement (register "name") for import.wast spec_tests = [os.path.join('spec', t) for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'spec'))) if t not in BLACKLIST] else: @@ -291,7 +306,7 @@ def run_spec_tests(): wast = os.path.join(options.binaryen_test, t) # skip checks for some tests - if os.path.basename(wast) in ['linking.wast', 'nop.wast', 'stack.wast', 'typecheck.wast', 'unwind.wast']: # FIXME + if os.path.basename(wast) in ['linking.wast', 'nop.wast', 'stack.wast', 'typecheck.wast', 'unwind.wast']: # FIXME continue def run_spec_test(wast): @@ -310,13 +325,17 @@ def run_spec_tests(): def check_expected(actual, expected): if expected and os.path.exists(expected): expected = open(expected).read() + # fix it up, our pretty (i32.const 83) must become compared to a homely 83 : i32 def fix(x): x = x.strip() - if not x: return x + if not x: + return x v, t = x.split(' : ') - if v.endswith('.'): v = v[:-1] # remove trailing '.' + if v.endswith('.'): + v = v[:-1] # remove trailing '.' return '(' + t + '.const ' + v + ')' + expected = '\n'.join(map(fix, expected.split('\n'))) print ' (using expected output)' actual = actual.strip() @@ -332,14 +351,14 @@ def run_spec_tests(): except Exception, e: if ('wasm-validator error' in str(e) or 'parse exception' in str(e)) and '.fail.' in t: print '<< test failed as expected >>' - continue # don't try all the binary format stuff TODO + continue # don't try all the binary format stuff TODO else: fail_with_error(str(e)) check_expected(actual, expected) # skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature - if os.path.basename(wast) in ['exports.wast']: # FIXME + if os.path.basename(wast) in ['exports.wast']: # FIXME continue # we must ignore some binary format splits @@ -350,7 +369,7 @@ def run_spec_tests(): # check binary format. here we can verify execution of the final result, no need for an output verification split_num = 0 - if os.path.basename(wast) not in []: # avoid some tests with things still being sorted out in the spec + if os.path.basename(wast) not in []: # avoid some tests with things still being sorted out in the spec actual = '' for module, asserts in split_wast(wast): skip = splits_to_skip.get(os.path.basename(wast)) or [] @@ -360,9 +379,10 @@ def run_spec_tests(): continue print ' testing split module', split_num split_num += 1 - with open('split.wast', 'w') as o: o.write(module + '\n' + '\n'.join(asserts)) - run_spec_test('split.wast') # before binary stuff - just check it's still ok split out - run_opt_test('split.wast') # also that our optimizer doesn't break on it + with open('split.wast', 'w') as o: + o.write(module + '\n' + '\n'.join(asserts)) + run_spec_test('split.wast') # before binary stuff - just check it's still ok split out + run_opt_test('split.wast') # also that our optimizer doesn't break on it result_wast = binary_format_check('split.wast', verify_final_result=False) # add the asserts, and verify that the test still passes open(result_wast, 'a').write('\n' + '\n'.join(asserts)) @@ -370,6 +390,7 @@ def run_spec_tests(): # compare all the outputs to the expected output check_expected(actual, os.path.join(options.binaryen_test, 'spec', 'expected-output', os.path.basename(wast) + '.log')) + def run_binaryen_js_tests(): if not MOZJS and not NODEJS: return @@ -378,7 +399,8 @@ def run_binaryen_js_tests(): print '\n[ checking binaryen.js testcases... ]\n' for s in sorted(os.listdir(os.path.join(options.binaryen_test, 'binaryen.js'))): - if not s.endswith('.js'): continue + if not s.endswith('.js'): + continue print s f = open('a.js', 'w') # avoid stdout/stderr ordering issues in some js shells - use just stdout @@ -393,21 +415,24 @@ def run_binaryen_js_tests(): test_src = open(test_path).read() f.write(test_src) f.close() + def test(engine): cmd = [engine, 'a.js'] out = run_command(cmd, stderr=subprocess.STDOUT) expected = open(os.path.join(options.binaryen_test, 'binaryen.js', s + '.txt')).read() if expected not in out: fail(out, expected) + # run in all possible shells if MOZJS: test(MOZJS) if NODEJS: - if node_has_wasm or not 'WebAssembly.' in test_src: + if node_has_wasm or 'WebAssembly.' not in test_src: test(NODEJS) else: print 'Skipping ' + test_path + ' because WebAssembly might not be supported' + def run_validator_tests(): print '\n[ running validation tests... ]\n' # Ensure the tests validate by default @@ -422,6 +447,7 @@ def run_validator_tests(): cmd = WASM_AS + ['--validate=none', os.path.join(options.binaryen_test, 'validator', 'invalid_return.wast')] run_command(cmd) + def run_torture_tests(): print '\n[ checking torture testcases... ]\n' @@ -465,12 +491,13 @@ def run_torture_tests(): else: del os.environ['BINARYEN_CORES'] + def run_vanilla_tests(): print '\n[ checking emcc WASM_BACKEND testcases...]\n' try: if has_vanilla_llvm: - os.environ['LLVM'] = BIN_DIR # use the vanilla LLVM + os.environ['LLVM'] = BIN_DIR # use the vanilla LLVM else: # if we did not set vanilla llvm, then we must set this env var to make emcc use the wasm backend. # (if we are using vanilla llvm, things should just work) @@ -483,15 +510,18 @@ def run_vanilla_tests(): subprocess.check_call(command) for c in sorted(os.listdir(os.path.join(options.binaryen_test, 'wasm_backend'))): - if not c.endswith('cpp'): continue + if not c.endswith('cpp'): + continue print '..', c base = c.replace('.cpp', '').replace('.c', '') expected = open(os.path.join(options.binaryen_test, 'wasm_backend', base + '.txt')).read() for opts in [[], ['-O1'], ['-O2']]: - only = [] if opts != ['-O1'] or '_only' not in base else ['-s', 'ONLY_MY_CODE=1'] # only my code is a hack we used early in wasm backend dev, which somehow worked, but only with -O1 + # only my code is a hack we used early in wasm backend dev, which somehow worked, but only with -O1 + only = [] if opts != ['-O1'] or '_only' not in base else ['-s', 'ONLY_MY_CODE=1'] command = [VANILLA_EMCC, '-o', 'a.wasm.js', os.path.join(options.binaryen_test, 'wasm_backend', c)] + opts + only print '....' + ' '.join(command) - if os.path.exists('a.wasm.js'): os.unlink('a.wasm.js') + if os.path.exists('a.wasm.js'): + os.unlink('a.wasm.js') subprocess.check_call(command) if NODEJS: print ' (check in node)' @@ -505,6 +535,7 @@ def run_vanilla_tests(): else: del os.environ['EMCC_WASM_BACKEND'] + def run_gcc_torture_tests(): print '\n[ checking native gcc testcases...]\n' if not NATIVECC or not NATIVEXX: @@ -521,7 +552,8 @@ def run_gcc_torture_tests(): continue print ' (will check trace in ', t, ')' src = 'trace.cpp' - with open(src, 'w') as o: o.write(out) + with open(src, 'w') as o: + o.write(out) expected = os.path.join(options.binaryen_test, 'example', t + '.txt') else: src = os.path.join(options.binaryen_test, 'example', t) @@ -558,6 +590,7 @@ def run_gcc_torture_tests(): fail_if_not_identical_to_file(actual, expected) + def run_emscripten_tests(): print '\n[ checking wasm.js methods... ]\n' @@ -565,12 +598,12 @@ def run_emscripten_tests(): # check success and failure for simple modes, only success for combined/fallback ones for success in [1, 0] if ',' not in method_init else [1]: method = method_init - command = [EMCC, '-o', 'a.wasm.js', '-s', 'BINARYEN=1', os.path.join(options.binaryen_test, 'hello_world.c') ] + command = [EMCC, '-o', 'a.wasm.js', '-s', 'BINARYEN=1', os.path.join(options.binaryen_test, 'hello_world.c')] command += ['-s', 'BINARYEN_METHOD="' + method + '"'] print method, ' : ', ' '.join(command), ' => ', success subprocess.check_call(command) - see_polyfill = 'var WasmJS = ' in open('a.wasm.js').read() + see_polyfill = 'var WasmJS = ' in open('a.wasm.js').read() if method and 'interpret' not in method: assert not see_polyfill, 'verify polyfill was not added - we specified a method, and it does not need it' @@ -581,27 +614,28 @@ def run_emscripten_tests(): asm = open('a.wasm.asm.js').read() asm = asm.replace('"almost asm"', '"use asm"; var not_in_asm = [].length + (true || { x: 5 }.x);') asm = asm.replace("'almost asm'", '"use asm"; var not_in_asm = [].length + (true || { x: 5 }.x);') - with open('a.wasm.asm.js', 'w') as o: o.write(asm) + with open('a.wasm.asm.js', 'w') as o: + o.write(asm) if method.startswith('interpret-asm2wasm'): - delete_from_orbit('a.wasm.wast') # we should not need the .wast + delete_from_orbit('a.wasm.wast') # we should not need the .wast if not success: - break_cashew() # we need cashew + break_cashew() # we need cashew elif method.startswith('interpret-s-expr'): - delete_from_orbit('a.wasm.asm.js') # we should not need the .asm.js + delete_from_orbit('a.wasm.asm.js') # we should not need the .asm.js if not success: delete_from_orbit('a.wasm.wast') elif method.startswith('asmjs'): - delete_from_orbit('a.wasm.wast') # we should not need the .wast - break_cashew() # we don't use cashew, so ok to break it + delete_from_orbit('a.wasm.wast') # we should not need the .wast + break_cashew() # we don't use cashew, so ok to break it if not success: delete_from_orbit('a.wasm.js') elif method.startswith('interpret-binary'): - delete_from_orbit('a.wasm.wast') # we should not need the .wast - delete_from_orbit('a.wasm.asm.js') # we should not need the .asm.js + delete_from_orbit('a.wasm.wast') # we should not need the .wast + delete_from_orbit('a.wasm.asm.js') # we should not need the .asm.js if not success: delete_from_orbit('a.wasm.wasm') else: - 1/0 + 1 / 0 if NODEJS: proc = subprocess.Popen([NODEJS, 'a.wasm.js'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() @@ -612,6 +646,7 @@ def run_emscripten_tests(): assert proc.returncode != 0, err assert 'hello, world!' not in out, out + # Run all the tests def main(): run_help_tests() @@ -653,8 +688,8 @@ def main(): if num_failures > 0: print '\n[ ' + str(num_failures) + ' failures! ]' - sys.exit(num_failures) + return num_failures -if __name__ == '__main__': - main() +if __name__ == '__main__': + sys.exit(main()) |