diff options
Diffstat (limited to 'scripts/test')
| -rwxr-xr-x | scripts/test/__init__.py | 17 | ||||
| -rwxr-xr-x | scripts/test/s2wasm.py | 103 | ||||
| -rw-r--r-- | scripts/test/shared.py | 406 | ||||
| -rwxr-xr-x | scripts/test/support.py | 158 |
4 files changed, 684 insertions, 0 deletions
diff --git a/scripts/test/__init__.py b/scripts/test/__init__.py new file mode 100755 index 000000000..8db5bb0bf --- /dev/null +++ b/scripts/test/__init__.py @@ -0,0 +1,17 @@ +#! /usr/bin/env python + +# Copyright 2015 WebAssembly Community Group participants +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Empty __init__.py file: Python treats the directory as containing a package. diff --git a/scripts/test/s2wasm.py b/scripts/test/s2wasm.py new file mode 100755 index 000000000..02cb5f147 --- /dev/null +++ b/scripts/test/s2wasm.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +# Copyright 2016 WebAssembly Community Group participants +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from support import run_command +from shared import ( + fail, fail_with_error, fail_if_not_contained, + options, S2WASM, WASM_SHELL +) + + +def test_s2wasm(): + print '\n[ checking .s testcases... ]\n' + + for dot_s_dir in ['dot_s', 'llvm_autogenerated']: + dot_s_path = os.path.join(options.binaryen_test, dot_s_dir) + for s in sorted(os.listdir(dot_s_path)): + if not s.endswith('.s'): + continue + print '..', s + wasm = s.replace('.s', '.wast') + full = os.path.join(options.binaryen_test, dot_s_dir, s) + stack_alloc = (['--allocate-stack=1024'] + if dot_s_dir == 'llvm_autogenerated' + else []) + cmd = S2WASM + [full, '--emscripten-glue'] + stack_alloc + if s.startswith('start_'): + cmd.append('--start') + actual = run_command(cmd) + + # verify output + expected_file = os.path.join(options.binaryen_test, dot_s_dir, wasm) + if not os.path.exists(expected_file): + print actual + fail_with_error('output ' + expected_file + ' does not exist') + expected = open(expected_file, 'rb').read() + if actual != expected: + fail(actual, expected) + + # verify with options + cmd = S2WASM + [full, '--global-base=1024'] + stack_alloc + run_command(cmd) + + # run wasm-shell on the .wast to verify that it parses + cmd = WASM_SHELL + [expected_file] + run_command(cmd) + + +def test_linker(): + print '\n[ running linker tests... ]\n' + # The {main,foo,bar,baz}.s files were created by running clang over the + # respective c files. The foobar.bar archive was created by running: + # llvm-ar -format=gnu rc foobar.a quux.s foo.s bar.s baz.s + cmd = S2WASM + [ + os.path.join(options.binaryen_test, 'linker', 'main.s'), '-l', + os.path.join(options.binaryen_test, 'linker', 'archive', 'foobar.a')] + output = run_command(cmd) + # foo should come from main.s and return 42 + fail_if_not_contained(output, '(func $foo') + fail_if_not_contained(output, '(i32.const 42)') + # bar should be linked in from bar.s + fail_if_not_contained(output, '(func $bar') + # quux should be linked in from bar.s even though it comes before bar.s in + # the archive + fail_if_not_contained(output, '(func $quux') + # baz should not be linked in at all + if 'baz' in output: + fail_with_error('output should not contain "baz": ' + output) + + # Test an archive using a string table + cmd = S2WASM + [ + os.path.join(options.binaryen_test, 'linker', 'main.s'), '-l', + os.path.join(options.binaryen_test, 'linker', 'archive', 'barlong.a')] + output = run_command(cmd) + # bar should be linked from the archive + fail_if_not_contained(output, '(func $bar') + + # Test exporting memory growth function + cmd = S2WASM + [ + os.path.join(options.binaryen_test, 'linker', 'main.s'), + '--emscripten-glue', '--allow-memory-growth'] + output = run_command(cmd) + fail_if_not_contained( + output, '(export "__growWasmMemory" (func $__growWasmMemory))') + fail_if_not_contained(output, '(func $__growWasmMemory (param $newSize i32)') + + +if __name__ == '__main__': + test_s2wasm() + test_linker() diff --git a/scripts/test/shared.py b/scripts/test/shared.py new file mode 100644 index 000000000..5aff20091 --- /dev/null +++ b/scripts/test/shared.py @@ -0,0 +1,406 @@ +import argparse +import difflib +import os +import shutil +import subprocess +import sys +import urllib2 + + +usage_str = ("usage: 'python check.py [options]'\n\n" + "Runs the Binaryen test suite.") +parser = argparse.ArgumentParser(description=usage_str) +parser.add_argument( + '--torture', dest='torture', action='store_true', default=True, + help='Chooses whether to run the torture testcases. Default: true.') +parser.add_argument( + '--no-torture', dest='torture', action='store_false', + help='Disables running the torture testcases.') +parser.add_argument( + '--only-prepare', dest='only_prepare', action='store_true', default=False, + help='If enabled, only fetches the waterfall build. Default: false.') +parser.add_argument( + # Backwards compatibility + '--only_prepare', dest='only_prepare', action='store_true', default=False, + help='If enabled, only fetches the waterfall build. Default: false.') +parser.add_argument( + '--test-waterfall', dest='test_waterfall', action='store_true', + default=True, + help=('If enabled, fetches and tests the LLVM waterfall builds.' + ' Default: true.')) +parser.add_argument( + '--no-test-waterfall', dest='test_waterfall', action='store_false', + help='Disables downloading and testing of the LLVM waterfall builds.') +parser.add_argument( + '--abort-on-first-failure', dest='abort_on_first_failure', + action='store_true', default=True, + help=('Specifies whether to halt test suite execution on first test error.' + ' Default: true.')) +parser.add_argument( + '--no-abort-on-first-failure', dest='abort_on_first_failure', + action='store_false', + help=('If set, the whole test suite will run to completion independent of' + ' earlier errors.')) +parser.add_argument( + '--run-gcc-tests', dest='run_gcc_tests', action='store_true', default=True, + help=('Chooses whether to run the tests that require building with native' + ' GCC. Default: true.')) +parser.add_argument( + '--no-run-gcc-tests', dest='run_gcc_tests', action='store_false', + help='If set, disables the native GCC tests.') + +parser.add_argument( + '--interpreter', dest='interpreter', default='', + help='Specifies the wasm interpreter executable to run tests on.') +parser.add_argument( + '--binaryen-bin', dest='binaryen_bin', default='', + help=('Specifies a path to where the built Binaryen executables reside at.' + ' Default: bin/ of current directory (i.e. assume an in-tree build).' + ' If not specified, the environment variable BINARYEN_ROOT= can also' + ' be used to adjust this.')) +parser.add_argument( + '--binaryen-root', dest='binaryen_root', default='', + help=('Specifies a path to the root of the Binaryen repository tree.' + ' Default: the directory where this file check.py resides.')) +parser.add_argument( + '--valgrind', dest='valgrind', default='', + help=('Specifies a path to Valgrind tool, which will be used to validate' + ' execution if specified. (Pass --valgrind=valgrind to search in' + ' PATH)')) +parser.add_argument( + '--valgrind-full-leak-check', dest='valgrind_full_leak_check', + action='store_true', default=False, + help=('If specified, all unfreed (but still referenced) pointers at the' + ' end of execution are considered memory leaks. Default: disabled.')) + +parser.add_argument( + 'positional_args', metavar='tests', nargs=argparse.REMAINDER, + help='Names specific tests to run.') +options = parser.parse_args() +requested = options.positional_args + +num_failures = 0 +warnings = [] + + +def warn(text): + global warnings + warnings.append(text) + print 'warning:', text + + +# setup + +# Locate Binaryen build artifacts directory (bin/ by default) +if not options.binaryen_bin: + if os.environ.get('BINARYEN_ROOT'): + if os.path.isdir(os.path.join(os.environ.get('BINARYEN_ROOT'), 'bin')): + options.binaryen_bin = os.path.join( + os.environ.get('BINARYEN_ROOT'), 'bin') + else: + options.binaryen_bin = os.environ.get('BINARYEN_ROOT') + else: + options.binaryen_bin = 'bin' + +wasm_dis_filenames = ['wasm-dis', 'wasm-dis.exe'] +if all(map(lambda f: not os.path.isfile(os.path.join(options.binaryen_bin, f)), + wasm_dis_filenames)): + warn('Binaryen not found (or has not been successfully built to bin/ ?') + +# Locate Binaryen source directory if not specified. +if not options.binaryen_root: + path_parts = os.path.abspath(__file__).split(os.path.sep) + options.binaryen_root = os.path.sep.join(path_parts[:-3]) + +options.binaryen_test = os.path.join(options.binaryen_root, 'test') + + +# Finds the given executable 'program' in PATH. +# Operates like the Unix tool 'which'. +def which(program): + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + if '.' not in fname: + if is_exe(exe_file + '.exe'): + return exe_file + '.exe' + if is_exe(exe_file + '.cmd'): + return exe_file + '.cmd' + if is_exe(exe_file + '.bat'): + return exe_file + '.bat' + + +WATERFALL_BUILD_DIR = os.path.join(options.binaryen_test, 'wasm-install') +BIN_DIR = os.path.abspath(os.path.join( + WATERFALL_BUILD_DIR, 'wasm-install', 'bin')) + +NATIVECC = (os.environ.get('CC') or which('mingw32-gcc') or + which('gcc') or which('clang')) +NATIVEXX = (os.environ.get('CXX') or which('mingw32-g++') or + which('g++') or which('clang++')) +NODEJS = which('nodejs') or which('node') +MOZJS = which('mozjs') +EMCC = which('emcc') + +WASM_OPT = [os.path.join(options.binaryen_bin, 'wasm-opt')] +WASM_AS = [os.path.join(options.binaryen_bin, 'wasm-as')] +WASM_DIS = [os.path.join(options.binaryen_bin, 'wasm-dis')] +ASM2WASM = [os.path.join(options.binaryen_bin, 'asm2wasm')] +WASM_SHELL = [os.path.join(options.binaryen_bin, 'wasm-shell')] +S2WASM = [os.path.join(options.binaryen_bin, 's2wasm')] + +S2WASM_EXE = S2WASM[0] +WASM_SHELL_EXE = WASM_SHELL[0] + + +def wrap_with_valgrind(cmd): + # Exit code 97 is arbitrary, used to easily detect when an error occurs that + # is detected by Valgrind. + valgrind = [options.valgrind, '--quiet', '--error-exitcode=97'] + if options.valgrind_full_leak_check: + valgrind += ['--leak-check=full', '--show-leak-kinds=all'] + return valgrind + cmd + + +if options.valgrind: + WASM_OPT = wrap_with_valgrind(WASM_OPT) + WASM_AS = wrap_with_valgrind(WASM_AS) + WASM_DIS = wrap_with_valgrind(WASM_DIS) + ASM2WASM = wrap_with_valgrind(ASM2WASM) + WASM_SHELL = wrap_with_valgrind(WASM_SHELL) + S2WASM = wrap_with_valgrind(S2WASM) + +os.environ['BINARYEN'] = os.getcwd() + + +def fetch_waterfall(): + rev = open(os.path.join(options.binaryen_test, 'revision')).read().strip() + buildername = {'linux2': 'linux', + 'darwin': 'mac', + 'win32': 'windows', + 'cygwin': 'windows'}[sys.platform] + try: + local_rev_path = os.path.join(options.binaryen_test, 'local-revision') + local_rev = open(local_rev_path).read().strip() + except: + local_rev = None + if local_rev == rev: + return + # fetch it + basename = 'wasm-binaries-' + rev + '.tbz2' + url = '/'.join(['https://storage.googleapis.com/wasm-llvm/builds', + buildername, rev, basename]) + print '(downloading waterfall %s: %s)' % (rev, url) + downloaded = urllib2.urlopen(url).read().strip() + fullname = os.path.join(options.binaryen_test, basename) + open(fullname, 'wb').write(downloaded) + print '(unpacking)' + if os.path.exists(WATERFALL_BUILD_DIR): + shutil.rmtree(WATERFALL_BUILD_DIR) + os.mkdir(WATERFALL_BUILD_DIR) + subprocess.check_call(['tar', '-xf', os.path.abspath(fullname)], + cwd=WATERFALL_BUILD_DIR) + print '(noting local revision)' + with open(os.path.join(options.binaryen_test, 'local-revision'), 'w') as o: + o.write(rev) + + +has_vanilla_llvm = False + + +def setup_waterfall(): + # if we can use the waterfall llvm, do so + global has_vanilla_llvm + CLANG = os.path.join(BIN_DIR, 'clang') + print 'trying waterfall clang at', CLANG + try: + subprocess.check_call([CLANG, '-v']) + has_vanilla_llvm = True + print '...success' + except Exception, e: + warn('could not run vanilla LLVM from waterfall: ' + str(e) + + ', looked for clang at ' + CLANG) + + +if options.test_waterfall: + fetch_waterfall() + setup_waterfall() + +if options.only_prepare: + print 'waterfall is fetched and setup, exiting since --only-prepare' + sys.exit(0) + +# external tools + +try: + subprocess.check_call( + [NODEJS, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +except: + NODEJS = None + warn('no node found (did not check proper js form)') + +try: + subprocess.check_call( + [MOZJS, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +except: + MOZJS = None + warn('no mozjs found (did not check native wasm support nor asm.js' + ' validation)') + +try: + subprocess.check_call( + [EMCC, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +except: + EMCC = None + warn('no emcc found (did not check non-vanilla emscripten/binaryen' + ' integration)') + +has_vanilla_emcc = False +try: + subprocess.check_call( + [os.path.join(options.binaryen_test, 'emscripten', 'emcc'), '--version'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + has_vanilla_emcc = True +except: + pass + + +# utilities + +# removes a file if it exists, using any and all ways of doing so +def delete_from_orbit(filename): + try: + os.unlink(filename) + except: + pass + if not os.path.exists(filename): + return + try: + shutil.rmtree(filename, ignore_errors=True) + except: + pass + if not os.path.exists(filename): + return + try: + import stat + os.chmod(filename, os.stat(filename).st_mode | stat.S_IWRITE) + + def remove_readonly_and_try_again(func, path, exc_info): + if not (os.stat(path).st_mode & stat.S_IWRITE): + os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE) + func(path) + else: + raise + shutil.rmtree(filename, onerror=remove_readonly_and_try_again) + except: + pass + + +def fail_with_error(msg): + global num_failures + try: + num_failures += 1 + raise Exception(msg) + except Exception, e: + print >> sys.stderr, str(e) + if options.abort_on_first_failure: + raise + + +def fail(actual, expected): + diff_lines = difflib.unified_diff( + expected.split('\n'), actual.split('\n'), + fromfile='expected', tofile='actual') + diff_str = ''.join([a.rstrip() + '\n' for a in diff_lines])[:] + fail_with_error("incorrect output, diff:\n\n%s" % diff_str) + + +def fail_if_not_identical(actual, expected): + if expected != actual: + fail(actual, expected) + + +def fail_if_not_contained(actual, expected): + if expected not in actual: + fail(actual, expected) + + +if len(requested) == 0: + tests = sorted(os.listdir(os.path.join(options.binaryen_test))) +else: + tests = requested[:] + +if not options.interpreter: + warn('no interpreter provided (did not test spec interpreter validation)') + +if not has_vanilla_emcc: + warn('no functional emcc submodule found') + + +# check utilities + +def binary_format_check(wast, verify_final_result=True, wasm_as_args=['-g'], + binary_suffix='.fromBinary'): + # checks we can convert the wast to binary and back + + print ' (binary format check)' + cmd = WASM_AS + [wast, '-o', 'a.wasm'] + wasm_as_args + print ' ', ' '.join(cmd) + if os.path.exists('a.wasm'): + os.unlink('a.wasm') + subprocess.check_call(cmd, stdout=subprocess.PIPE) + assert os.path.exists('a.wasm') + + cmd = WASM_DIS + ['a.wasm', '-o', 'ab.wast'] + print ' ', ' '.join(cmd) + if os.path.exists('ab.wast'): + os.unlink('ab.wast') + subprocess.check_call(cmd, stdout=subprocess.PIPE) + assert os.path.exists('ab.wast') + + # make sure it is a valid wast + cmd = WASM_OPT + ['ab.wast'] + print ' ', ' '.join(cmd) + subprocess.check_call(cmd, stdout=subprocess.PIPE) + + if verify_final_result: + expected = open(wast + binary_suffix).read() + actual = open('ab.wast').read() + if actual != expected: + fail(actual, expected) + + return 'ab.wast' + + +def minify_check(wast, verify_final_result=True): + # checks we can parse minified output + + print ' (minify check)' + cmd = WASM_OPT + [wast, '--print-minified'] + print ' ', ' '.join(cmd) + subprocess.check_call( + WASM_OPT + [wast, '--print-minified'], + stdout=open('a.wasm', 'w'), stderr=subprocess.PIPE) + assert os.path.exists('a.wasm') + subprocess.check_call( + WASM_OPT + ['a.wasm', '--print-minified'], + stdout=open('b.wasm', 'w'), stderr=subprocess.PIPE) + assert os.path.exists('b.wasm') + if verify_final_result: + expected = open('a.wasm').read() + actual = open('b.wasm').read() + if actual != expected: + fail(actual, expected) + if os.path.exists('a.wasm'): + os.unlink('a.wasm') + if os.path.exists('b.wasm'): + os.unlink('b.wasm') diff --git a/scripts/test/support.py b/scripts/test/support.py new file mode 100755 index 000000000..43762fffa --- /dev/null +++ b/scripts/test/support.py @@ -0,0 +1,158 @@ +#! /usr/bin/env python + +# Copyright 2016 WebAssembly Community Group participants +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import filecmp +import os +import shutil +import subprocess +import sys +import tempfile + + +def _open_archive(tarfile, tmp_dir): + with tempfile.TemporaryFile(mode='w+') as f: + try: + subprocess.check_call(['tar', '-xvf', tarfile], cwd=tmp_dir, stdout=f) + except: + f.seek(0) + sys.stderr.write(f.read()) + raise + return os.listdir(tmp_dir) + + +def _files_same(dir1, dir2, basenames): + diff = filecmp.cmpfiles(dir1, dir2, basenames) + return 0 == len(diff[1] + diff[2]) + + +def _dirs_same(dir1, dir2, basenames): + for d in basenames: + left = os.path.join(dir1, d) + right = os.path.join(dir2, d) + if not (os.path.isdir(left) and os.path.isdir(right)): + return False + diff = filecmp.dircmp(right, right) + if 0 != len(diff.left_only + diff.right_only + diff.diff_files + + diff.common_funny + diff.funny_files): + return False + return True + + +def _move_files(dirfrom, dirto, basenames): + for f in basenames: + from_file = os.path.join(dirfrom, f) + to_file = os.path.join(dirto, f) + if os.path.isfile(to_file): + os.path.remove(to_file) + shutil.move(from_file, to_file) + + +def _move_dirs(dirfrom, dirto, basenames): + for d in basenames: + from_dir = os.path.join(dirfrom, d) + to_dir = os.path.join(dirto, d) + if os.path.isdir(to_dir): + shutil.rmtree(to_dir) + shutil.move(from_dir, to_dir) + + +def untar(tarfile, outdir): + """Returns True if untar content differs from pre-existing outdir content.""" + tmpdir = tempfile.mkdtemp() + try: + untared = _open_archive(tarfile, tmpdir) + files = [f for f in untared if os.path.isfile(os.path.join(tmpdir, f))] + dirs = [d for d in untared if os.path.isdir(os.path.join(tmpdir, d))] + assert len(files) + len(dirs) == len(untared), 'Only files and directories' + if _files_same(tmpdir, outdir, files) and _dirs_same(tmpdir, outdir, dirs): + # Nothing new or different in the tarfile. + return False + # Some or all of the files / directories are new. + _move_files(tmpdir, outdir, files) + _move_dirs(tmpdir, outdir, dirs) + return True + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + + +def split_wast(wast): + # .wast files can contain multiple modules, and assertions for each one. + # this splits out a wast into [(module, assertions), ..] + # we ignore module invalidity tests here. + wast = open(wast).read() + ret = [] + + def to_end(j): + depth = 1 + while depth > 0 and j < len(wast): + if wast[j] == '"': + while 1: + j = wast.find('"', j + 1) + if wast[j - 1] == '\\': + continue + break + assert j > 0 + elif wast[j] == '(': + depth += 1 + elif wast[j] == ')': + depth -= 1 + elif wast[j] == ';' and wast[j + 1] == ';': + j = wast.find('\n', j) + j += 1 + return j + + i = 0 + while i >= 0: + start = wast.find('(', i) + if start >= 0 and wast[start + 1] == ';': + # block comment + i = wast.find(';)', start + 2) + assert i > 0, wast[start:] + i += 2 + continue + skip = wast.find(';', i) + if skip >= 0 and skip < start and skip + 1 < len(wast): + if wast[skip + 1] == ';': + i = wast.find('\n', i) + 1 + continue + if start < 0: + break + i = to_end(start + 1) + chunk = wast[start:i] + if chunk.startswith('(module'): + ret += [(chunk, [])] + elif chunk.startswith('(assert_invalid'): + continue + elif chunk.startswith(('(assert', '(invoke')): + ret[-1][1].append(chunk) + return ret + + +def run_command(cmd, expected_status=0, stderr=None, expected_err=None): + if expected_err is not None: + assert stderr == subprocess.PIPE or stderr is None,\ + "Can't redirect stderr if using expected_err" + stderr = subprocess.PIPE + print 'executing: ', ' '.join(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr) + out, err = proc.communicate() + if proc.returncode != expected_status: + raise Exception(('run_command failed', err)) + if expected_err is not None and err != expected_err: + raise Exception(('run_command unexpected stderr', + "expected '%s', actual '%s'" % (expected_err, err))) + return out |