diff options
author | Sam Clegg <sbc@chromium.org> | 2019-08-16 13:31:52 -0700 |
---|---|---|
committer | Guanzhong Chen <gzchen@google.com> | 2019-08-16 13:31:52 -0700 |
commit | c8a797d120a1413d993281c98268c1c2ee9f3f94 (patch) | |
tree | 30b0d00b8af103e1b8a751f292f0241a02af9a79 /scripts/fuzz_opt.py | |
parent | ee0564088c7f89814bf951cc5936aa096538e38f (diff) | |
download | binaryen-c8a797d120a1413d993281c98268c1c2ee9f3f94.tar.gz binaryen-c8a797d120a1413d993281c98268c1c2ee9f3f94.tar.bz2 binaryen-c8a797d120a1413d993281c98268c1c2ee9f3f94.zip |
Switch python indentation from 2-space to 4-space (#2299)
pep8 specifies 4 space indentation. The use of 2 spaces is, I believe
a historical anomaly where certain large organizations such as google
chose 2 over 4 and have yet to make the switch.
Since there isn't too much code in binaryen today it seems reasonable to
make the switch.
Diffstat (limited to 'scripts/fuzz_opt.py')
-rw-r--r-- | scripts/fuzz_opt.py | 788 |
1 files changed, 394 insertions, 394 deletions
diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index 9af65b09d..4591b4150 100644 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -47,49 +47,49 @@ LOG_LIMIT = 125 def in_binaryen(*args): - return os.path.join(options.binaryen_root, *args) + return os.path.join(options.binaryen_root, *args) def in_bin(tool): - return os.path.join(options.binaryen_root, 'bin', tool) + return os.path.join(options.binaryen_root, 'bin', tool) def random_size(): - return random.randint(1, INPUT_SIZE_LIMIT) + return random.randint(1, INPUT_SIZE_LIMIT) def run(cmd): - print(' '.join(cmd)[:LOG_LIMIT]) - return subprocess.check_output(cmd) + print(' '.join(cmd)[:LOG_LIMIT]) + return subprocess.check_output(cmd) def run_unchecked(cmd): - print(' '.join(cmd)[:LOG_LIMIT]) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0] + print(' '.join(cmd)[:LOG_LIMIT]) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0] def randomize_pass_debug(): - if random.random() < 0.125: - print('[pass-debug]') - os.environ['BINARYEN_PASS_DEBUG'] = '1' - else: - os.environ['BINARYEN_PASS_DEBUG'] = '0' - del os.environ['BINARYEN_PASS_DEBUG'] + if random.random() < 0.125: + print('[pass-debug]') + os.environ['BINARYEN_PASS_DEBUG'] = '1' + else: + os.environ['BINARYEN_PASS_DEBUG'] = '0' + del os.environ['BINARYEN_PASS_DEBUG'] def randomize_feature_opts(): - global FEATURE_OPTS - FEATURE_OPTS = CONSTANT_FEATURE_OPTS[:] - # half the time apply all the possible opts. this lets all test runners work at max - # capacity at least half the time, as otherwise if they need almost all the opts, the - # chance of getting them is exponentially small. - if random.random() < 0.5: - FEATURE_OPTS += POSSIBLE_FEATURE_OPTS - else: - for possible in POSSIBLE_FEATURE_OPTS: - if random.random() < 0.5: - FEATURE_OPTS.append(possible) - print('feature opts:', ' '.join(FEATURE_OPTS)) + global FEATURE_OPTS + FEATURE_OPTS = CONSTANT_FEATURE_OPTS[:] + # half the time apply all the possible opts. this lets all test runners work at max + # capacity at least half the time, as otherwise if they need almost all the opts, the + # chance of getting them is exponentially small. + if random.random() < 0.5: + FEATURE_OPTS += POSSIBLE_FEATURE_OPTS + else: + for possible in POSSIBLE_FEATURE_OPTS: + if random.random() < 0.5: + FEATURE_OPTS.append(possible) + print('feature opts:', ' '.join(FEATURE_OPTS)) # Test outputs we want to ignore are marked this way. @@ -97,59 +97,59 @@ IGNORE = '[binaryen-fuzzer-ignore]' def compare(x, y, context): - if x != y and x != IGNORE and y != IGNORE: - message = ''.join([a + '\n' for a in difflib.unified_diff(x.splitlines(), y.splitlines(), fromfile='expected', tofile='actual')]) - raise Exception(context + " comparison error, expected to have '%s' == '%s', diff:\n\n%s" % ( - x, y, - message - )) + if x != y and x != IGNORE and y != IGNORE: + message = ''.join([a + '\n' for a in difflib.unified_diff(x.splitlines(), y.splitlines(), fromfile='expected', tofile='actual')]) + raise Exception(context + " comparison error, expected to have '%s' == '%s', diff:\n\n%s" % ( + x, y, + message + )) def fix_output(out): - # large doubles may print slightly different on different VMs - def fix_double(x): - x = x.group(1) - if 'nan' in x or 'NaN' in x: - x = 'nan' - else: - x = x.replace('Infinity', 'inf') - x = str(float(x)) - return 'f64.const ' + x - out = re.sub(r'f64\.const (-?[nanN:abcdefxIity\d+-.]+)', fix_double, out) + # large doubles may print slightly different on different VMs + def fix_double(x): + x = x.group(1) + if 'nan' in x or 'NaN' in x: + x = 'nan' + else: + x = x.replace('Infinity', 'inf') + x = str(float(x)) + return 'f64.const ' + x + out = re.sub(r'f64\.const (-?[nanN:abcdefxIity\d+-.]+)', fix_double, out) - # mark traps from wasm-opt as exceptions, even though they didn't run in a vm - out = out.replace('[trap ', 'exception: [trap ') + # mark traps from wasm-opt as exceptions, even though they didn't run in a vm + out = out.replace('[trap ', 'exception: [trap ') - # exceptions may differ when optimizing, but an exception should occur. so ignore their types - # also js engines print them out slightly differently - return '\n'.join(map(lambda x: ' *exception*' if 'exception' in x else x, out.splitlines())) + # exceptions may differ when optimizing, but an exception should occur. so ignore their types + # also js engines print them out slightly differently + return '\n'.join(map(lambda x: ' *exception*' if 'exception' in x else x, out.splitlines())) def fix_spec_output(out): - out = fix_output(out) - # spec shows a pointer when it traps, remove that - out = '\n'.join(map(lambda x: x if 'runtime trap' not in x else x[x.find('runtime trap'):], out.splitlines())) - # https://github.com/WebAssembly/spec/issues/543 , float consts are messed up - out = '\n'.join(map(lambda x: x if 'f32' not in x and 'f64' not in x else '', out.splitlines())) - return out + out = fix_output(out) + # spec shows a pointer when it traps, remove that + out = '\n'.join(map(lambda x: x if 'runtime trap' not in x else x[x.find('runtime trap'):], out.splitlines())) + # https://github.com/WebAssembly/spec/issues/543 , float consts are messed up + out = '\n'.join(map(lambda x: x if 'f32' not in x and 'f64' not in x else '', out.splitlines())) + return out def run_vm(cmd): - # ignore some vm assertions, if bugs have already been filed - known_issues = [ - 'local count too large', # ignore this; can be caused by flatten, ssa, etc. passes - 'liftoff-assembler.cc, line 239\n', # https://bugs.chromium.org/p/v8/issues/detail?id=8631 - 'liftoff-assembler.cc, line 245\n', # https://bugs.chromium.org/p/v8/issues/detail?id=8631 - 'liftoff-register.h, line 86\n', # https://bugs.chromium.org/p/v8/issues/detail?id=8632 - ] - try: - return run(cmd) - except subprocess.CalledProcessError: - output = run_unchecked(cmd) - for issue in known_issues: - if issue in output: - return IGNORE - raise + # ignore some vm assertions, if bugs have already been filed + known_issues = [ + 'local count too large', # ignore this; can be caused by flatten, ssa, etc. passes + 'liftoff-assembler.cc, line 239\n', # https://bugs.chromium.org/p/v8/issues/detail?id=8631 + 'liftoff-assembler.cc, line 245\n', # https://bugs.chromium.org/p/v8/issues/detail?id=8631 + 'liftoff-register.h, line 86\n', # https://bugs.chromium.org/p/v8/issues/detail?id=8632 + ] + try: + return run(cmd) + except subprocess.CalledProcessError: + output = run_unchecked(cmd) + for issue in known_issues: + if issue in output: + return IGNORE + raise MAX_INTERPRETER_ENV_VAR = 'BINARYEN_MAX_INTERPRETER_DEPTH' @@ -157,101 +157,101 @@ MAX_INTERPRETER_DEPTH = 1000 def run_bynterp(wasm, args): - # increase the interpreter stack depth, to test more things - os.environ[MAX_INTERPRETER_ENV_VAR] = str(MAX_INTERPRETER_DEPTH) - try: - return run_vm([in_bin('wasm-opt'), wasm] + FEATURE_OPTS + args) - finally: - del os.environ['BINARYEN_MAX_INTERPRETER_DEPTH'] + # increase the interpreter stack depth, to test more things + os.environ[MAX_INTERPRETER_ENV_VAR] = str(MAX_INTERPRETER_DEPTH) + try: + return run_vm([in_bin('wasm-opt'), wasm] + FEATURE_OPTS + args) + finally: + del os.environ['BINARYEN_MAX_INTERPRETER_DEPTH'] def run_d8(wasm): - return run_vm(['d8'] + V8_OPTS + [in_binaryen('scripts', 'fuzz_shell.js'), '--', wasm]) + return run_vm(['d8'] + V8_OPTS + [in_binaryen('scripts', 'fuzz_shell.js'), '--', wasm]) # There are two types of test case handlers: -# * get_commands() users: these return a list of commands to run (for example, "run this wasm-opt -# command, then that one"). The calling code gets and runs those commands on the test wasm -# file, and has enough information and control to be able to perform auto-reduction of any -# bugs found. -# * Totally generic: These receive the input pattern, a wasm generated from it, and a wasm -# optimized from that, and can then do anything it wants with those. +# * get_commands() users: these return a list of commands to run (for example, "run this wasm-opt +# command, then that one"). The calling code gets and runs those commands on the test wasm +# file, and has enough information and control to be able to perform auto-reduction of any +# bugs found. +# * Totally generic: These receive the input pattern, a wasm generated from it, and a wasm +# optimized from that, and can then do anything it wants with those. class TestCaseHandler: - # If the core handle_pair() method is not overridden, it calls handle_single() - # on each of the pair. That is useful if you just want the two wasms, and don't - # care about their relationship - def handle_pair(self, input, before_wasm, after_wasm, opts): - self.handle(before_wasm) - self.handle(after_wasm) + # If the core handle_pair() method is not overridden, it calls handle_single() + # on each of the pair. That is useful if you just want the two wasms, and don't + # care about their relationship + def handle_pair(self, input, before_wasm, after_wasm, opts): + self.handle(before_wasm) + self.handle(after_wasm) - def can_run_on_feature_opts(self, feature_opts): - return True + def can_run_on_feature_opts(self, feature_opts): + return True # Run VMs and compare results class CompareVMs(TestCaseHandler): - def handle_pair(self, input, before_wasm, after_wasm, opts): - run([in_bin('wasm-opt'), before_wasm, '--emit-js-wrapper=a.js', '--emit-spec-wrapper=a.wat'] + FEATURE_OPTS) - run([in_bin('wasm-opt'), after_wasm, '--emit-js-wrapper=b.js', '--emit-spec-wrapper=b.wat'] + FEATURE_OPTS) - before = self.run_vms('a.js', before_wasm) - after = self.run_vms('b.js', after_wasm) - self.compare_vs(before, after) - - def run_vms(self, js, wasm): - results = [] - results.append(fix_output(run_bynterp(wasm, ['--fuzz-exec-before']))) - results.append(fix_output(run_vm(['d8', js] + V8_OPTS + ['--', wasm]))) - - # append to add results from VMs - # results += [fix_output(run_vm(['d8', js] + V8_OPTS + ['--', wasm]))] - # results += [fix_output(run_vm([os.path.expanduser('~/.jsvu/jsc'), js, '--', wasm]))] - # spec has no mechanism to not halt on a trap. so we just check until the first trap, basically - # run(['../spec/interpreter/wasm', wasm]) - # results += [fix_spec_output(run_unchecked(['../spec/interpreter/wasm', wasm, '-e', open(prefix + 'wat').read()]))] - - if len(results) == 0: - results = [0] - - # NaNs are a source of nondeterminism between VMs; don't compare them - if not NANS: - first = results[0] - for i in range(len(results)): - compare(first, results[i], 'CompareVMs at ' + str(i)) - - return results - - def compare_vs(self, before, after): - for i in range(len(before)): - compare(before[i], after[i], 'CompareVMs at ' + str(i)) - # with nans, we can only compare the binaryen interpreter to itself - if NANS: - break - - def can_run_on_feature_opts(self, feature_opts): - return all([x in feature_opts for x in ['--disable-simd']]) + def handle_pair(self, input, before_wasm, after_wasm, opts): + run([in_bin('wasm-opt'), before_wasm, '--emit-js-wrapper=a.js', '--emit-spec-wrapper=a.wat'] + FEATURE_OPTS) + run([in_bin('wasm-opt'), after_wasm, '--emit-js-wrapper=b.js', '--emit-spec-wrapper=b.wat'] + FEATURE_OPTS) + before = self.run_vms('a.js', before_wasm) + after = self.run_vms('b.js', after_wasm) + self.compare_vs(before, after) + + def run_vms(self, js, wasm): + results = [] + results.append(fix_output(run_bynterp(wasm, ['--fuzz-exec-before']))) + results.append(fix_output(run_vm(['d8', js] + V8_OPTS + ['--', wasm]))) + + # append to add results from VMs + # results += [fix_output(run_vm(['d8', js] + V8_OPTS + ['--', wasm]))] + # results += [fix_output(run_vm([os.path.expanduser('~/.jsvu/jsc'), js, '--', wasm]))] + # spec has no mechanism to not halt on a trap. so we just check until the first trap, basically + # run(['../spec/interpreter/wasm', wasm]) + # results += [fix_spec_output(run_unchecked(['../spec/interpreter/wasm', wasm, '-e', open(prefix + 'wat').read()]))] + + if len(results) == 0: + results = [0] + + # NaNs are a source of nondeterminism between VMs; don't compare them + if not NANS: + first = results[0] + for i in range(len(results)): + compare(first, results[i], 'CompareVMs at ' + str(i)) + + return results + + def compare_vs(self, before, after): + for i in range(len(before)): + compare(before[i], after[i], 'CompareVMs at ' + str(i)) + # with nans, we can only compare the binaryen interpreter to itself + if NANS: + break + + def can_run_on_feature_opts(self, feature_opts): + return all([x in feature_opts for x in ['--disable-simd']]) # Fuzz the interpreter with --fuzz-exec. This tests everything in a single command (no # two separate binaries) so it's easy to reproduce. class FuzzExec(TestCaseHandler): - def get_commands(self, wasm, opts, random_seed): - return [ - '%(MAX_INTERPRETER_ENV_VAR)s=%(MAX_INTERPRETER_DEPTH)d %(wasm_opt)s --fuzz-exec --fuzz-binary %(opts)s %(wasm)s' % { - 'MAX_INTERPRETER_ENV_VAR': MAX_INTERPRETER_ENV_VAR, - 'MAX_INTERPRETER_DEPTH': MAX_INTERPRETER_DEPTH, - 'wasm_opt': in_bin('wasm-opt'), - 'opts': ' '.join(opts), - 'wasm': wasm - } - ] + def get_commands(self, wasm, opts, random_seed): + return [ + '%(MAX_INTERPRETER_ENV_VAR)s=%(MAX_INTERPRETER_DEPTH)d %(wasm_opt)s --fuzz-exec --fuzz-binary %(opts)s %(wasm)s' % { + 'MAX_INTERPRETER_ENV_VAR': MAX_INTERPRETER_ENV_VAR, + 'MAX_INTERPRETER_DEPTH': MAX_INTERPRETER_DEPTH, + 'wasm_opt': in_bin('wasm-opt'), + 'opts': ' '.join(opts), + 'wasm': wasm + } + ] # As FuzzExec, but without a separate invocation. This can find internal bugs with generating # the IR (which might be worked around by writing it and then reading it). class FuzzExecImmediately(TestCaseHandler): - def handle_pair(self, input, before_wasm, after_wasm, opts): - # fuzz binaryen interpreter itself. separate invocation so result is easily reduceable - run_bynterp(before_wasm, ['--fuzz-exec', '--fuzz-binary'] + opts) + def handle_pair(self, input, before_wasm, after_wasm, opts): + # fuzz binaryen interpreter itself. separate invocation so result is easily reduceable + run_bynterp(before_wasm, ['--fuzz-exec', '--fuzz-binary'] + opts) # Check for determinism - the same command must have the same output. @@ -259,274 +259,274 @@ class FuzzExecImmediately(TestCaseHandler): # for something that autoreduction won't help with anyhow (nondeterminism is very # hard to reduce). class CheckDeterminism(TestCaseHandler): - def handle_pair(self, input, before_wasm, after_wasm, opts): - # check for determinism - run([in_bin('wasm-opt'), before_wasm, '-o', 'b1.wasm'] + opts) - run([in_bin('wasm-opt'), before_wasm, '-o', 'b2.wasm'] + opts) - assert open('b1.wasm').read() == open('b2.wasm').read(), 'output must be deterministic' + def handle_pair(self, input, before_wasm, after_wasm, opts): + # check for determinism + run([in_bin('wasm-opt'), before_wasm, '-o', 'b1.wasm'] + opts) + run([in_bin('wasm-opt'), before_wasm, '-o', 'b2.wasm'] + opts) + assert open('b1.wasm').read() == open('b2.wasm').read(), 'output must be deterministic' class Wasm2JS(TestCaseHandler): - def handle_pair(self, input, before_wasm, after_wasm, opts): - compare(self.run(before_wasm), self.run(after_wasm), 'Wasm2JS') - - def run(self, wasm): - # TODO: wasm2js does not handle nans precisely, and does not - # handle oob loads etc. with traps, should we use - # FUZZ_OPTS += ['--no-fuzz-nans'] - # FUZZ_OPTS += ['--no-fuzz-oob'] - # ? - wrapper = run([in_bin('wasm-opt'), wasm, '--emit-js-wrapper=/dev/stdout'] + FEATURE_OPTS) - cmd = [in_bin('wasm2js'), wasm, '--emscripten'] - if random.random() < 0.5: - cmd += ['-O'] - main = run(cmd + FEATURE_OPTS) - with open(os.path.join(options.binaryen_root, 'scripts', 'wasm2js.js')) as f: - glue = f.read() - with open('js.js', 'w') as f: - f.write(glue) - f.write(main) - f.write(wrapper) - out = fix_output(run_vm([NODEJS, 'js.js', 'a.wasm'])) - if 'exception' in out: - # exception, so ignoring - wasm2js does not have normal wasm trapping, so opts can eliminate a trap - out = IGNORE - return out - - def can_run_on_feature_opts(self, feature_opts): - return all([x in feature_opts for x in ['--disable-exception-handling', '--disable-simd', '--disable-threads', '--disable-bulk-memory', '--disable-nontrapping-float-to-int', '--disable-tail-call', '--disable-sign-ext']]) + def handle_pair(self, input, before_wasm, after_wasm, opts): + compare(self.run(before_wasm), self.run(after_wasm), 'Wasm2JS') + + def run(self, wasm): + # TODO: wasm2js does not handle nans precisely, and does not + # handle oob loads etc. with traps, should we use + # FUZZ_OPTS += ['--no-fuzz-nans'] + # FUZZ_OPTS += ['--no-fuzz-oob'] + # ? + wrapper = run([in_bin('wasm-opt'), wasm, '--emit-js-wrapper=/dev/stdout'] + FEATURE_OPTS) + cmd = [in_bin('wasm2js'), wasm, '--emscripten'] + if random.random() < 0.5: + cmd += ['-O'] + main = run(cmd + FEATURE_OPTS) + with open(os.path.join(options.binaryen_root, 'scripts', 'wasm2js.js')) as f: + glue = f.read() + with open('js.js', 'w') as f: + f.write(glue) + f.write(main) + f.write(wrapper) + out = fix_output(run_vm([NODEJS, 'js.js', 'a.wasm'])) + if 'exception' in out: + # exception, so ignoring - wasm2js does not have normal wasm trapping, so opts can eliminate a trap + out = IGNORE + return out + + def can_run_on_feature_opts(self, feature_opts): + return all([x in feature_opts for x in ['--disable-exception-handling', '--disable-simd', '--disable-threads', '--disable-bulk-memory', '--disable-nontrapping-float-to-int', '--disable-tail-call', '--disable-sign-ext']]) class Asyncify(TestCaseHandler): - def handle_pair(self, input, before_wasm, after_wasm, opts): - # we must legalize in order to run in JS - run([in_bin('wasm-opt'), before_wasm, '--legalize-js-interface', '-o', before_wasm] + FEATURE_OPTS) - run([in_bin('wasm-opt'), after_wasm, '--legalize-js-interface', '-o', after_wasm] + FEATURE_OPTS) - before = fix_output(run_d8(before_wasm)) - after = fix_output(run_d8(after_wasm)) - - # TODO: also something that actually does async sleeps in the code, say - # on the logging commands? - # --remove-unused-module-elements removes the asyncify intrinsics, which are not valid to call - - def do_asyncify(wasm): - cmd = [in_bin('wasm-opt'), wasm, '--asyncify', '-o', 't.wasm'] - if random.random() < 0.5: - cmd += ['--optimize-level=%d' % random.randint(1, 3)] - if random.random() < 0.5: - cmd += ['--shrink-level=%d' % random.randint(1, 2)] - cmd += FEATURE_OPTS - run(cmd) - out = run_d8('t.wasm') - # emit some status logging from asyncify - print(out.splitlines()[-1]) - # ignore the output from the new asyncify API calls - the ones with asserts will trap, too - for ignore in ['[fuzz-exec] calling $asyncify_start_unwind\nexception!\n', - '[fuzz-exec] calling $asyncify_start_unwind\n', - '[fuzz-exec] calling $asyncify_start_rewind\nexception!\n', - '[fuzz-exec] calling $asyncify_start_rewind\n', - '[fuzz-exec] calling $asyncify_stop_rewind\n', - '[fuzz-exec] calling $asyncify_stop_unwind\n']: - out = out.replace(ignore, '') - out = '\n'.join([l for l in out.splitlines() if 'asyncify: ' not in l]) - return fix_output(out) - - before_asyncify = do_asyncify(before_wasm) - after_asyncify = do_asyncify(after_wasm) - - compare(before, after, 'Asyncify (before/after)') - compare(before, before_asyncify, 'Asyncify (before/before_asyncify)') - compare(before, after_asyncify, 'Asyncify (before/after_asyncify)') - - def can_run_on_feature_opts(self, feature_opts): - return all([x in feature_opts for x in ['--disable-exception-handling', '--disable-simd', '--disable-tail-call']]) + def handle_pair(self, input, before_wasm, after_wasm, opts): + # we must legalize in order to run in JS + run([in_bin('wasm-opt'), before_wasm, '--legalize-js-interface', '-o', before_wasm] + FEATURE_OPTS) + run([in_bin('wasm-opt'), after_wasm, '--legalize-js-interface', '-o', after_wasm] + FEATURE_OPTS) + before = fix_output(run_d8(before_wasm)) + after = fix_output(run_d8(after_wasm)) + + # TODO: also something that actually does async sleeps in the code, say + # on the logging commands? + # --remove-unused-module-elements removes the asyncify intrinsics, which are not valid to call + + def do_asyncify(wasm): + cmd = [in_bin('wasm-opt'), wasm, '--asyncify', '-o', 't.wasm'] + if random.random() < 0.5: + cmd += ['--optimize-level=%d' % random.randint(1, 3)] + if random.random() < 0.5: + cmd += ['--shrink-level=%d' % random.randint(1, 2)] + cmd += FEATURE_OPTS + run(cmd) + out = run_d8('t.wasm') + # emit some status logging from asyncify + print(out.splitlines()[-1]) + # ignore the output from the new asyncify API calls - the ones with asserts will trap, too + for ignore in ['[fuzz-exec] calling $asyncify_start_unwind\nexception!\n', + '[fuzz-exec] calling $asyncify_start_unwind\n', + '[fuzz-exec] calling $asyncify_start_rewind\nexception!\n', + '[fuzz-exec] calling $asyncify_start_rewind\n', + '[fuzz-exec] calling $asyncify_stop_rewind\n', + '[fuzz-exec] calling $asyncify_stop_unwind\n']: + out = out.replace(ignore, '') + out = '\n'.join([l for l in out.splitlines() if 'asyncify: ' not in l]) + return fix_output(out) + + before_asyncify = do_asyncify(before_wasm) + after_asyncify = do_asyncify(after_wasm) + + compare(before, after, 'Asyncify (before/after)') + compare(before, before_asyncify, 'Asyncify (before/before_asyncify)') + compare(before, after_asyncify, 'Asyncify (before/after_asyncify)') + + def can_run_on_feature_opts(self, feature_opts): + return all([x in feature_opts for x in ['--disable-exception-handling', '--disable-simd', '--disable-tail-call']]) # The global list of all test case handlers testcase_handlers = [ - FuzzExec(), - CompareVMs(), - CheckDeterminism(), - Wasm2JS(), - Asyncify(), - FuzzExecImmediately(), + FuzzExec(), + CompareVMs(), + CheckDeterminism(), + Wasm2JS(), + Asyncify(), + FuzzExecImmediately(), ] # Do one test, given an input file for -ttf and some optimizations to run def test_one(random_input, opts): - randomize_pass_debug() - randomize_feature_opts() - - run([in_bin('wasm-opt'), random_input, '-ttf', '-o', 'a.wasm'] + FUZZ_OPTS + FEATURE_OPTS) - wasm_size = os.stat('a.wasm').st_size - bytes = wasm_size - print('pre wasm size:', wasm_size) - - # first, run all handlers that use get_commands(). those don't need the second wasm in the - # pair, since they all they do is return their commands, and expect us to run them, and - # those commands do the actual testing, by operating on the original input wasm file. by - # fuzzing the get_commands() ones first we can find bugs in creating the second wasm (that - # has the opts run on it) before we try to create it later down for the passes that - # expect to get it as one of their inputs. - for testcase_handler in testcase_handlers: - if testcase_handler.can_run_on_feature_opts(FEATURE_OPTS): - if hasattr(testcase_handler, 'get_commands'): - print('running testcase handler:', testcase_handler.__class__.__name__) - # if the testcase handler supports giving us a list of commands, then we can get those commands - # and use them to do useful things like automatic reduction. in this case we give it the input - # wasm plus opts and a random seed (if it needs any internal randomness; we want to have the same - # value there if we reduce). - random_seed = random.random() - - # gets commands from the handler, for a given set of optimizations. this is all the commands - # needed to run the testing that that handler wants to do. - def get_commands(opts): - return testcase_handler.get_commands(wasm='a.wasm', opts=opts + FUZZ_OPTS + FEATURE_OPTS, random_seed=random_seed) - - def write_commands_and_test(opts): - commands = get_commands(opts) - write_commands(commands, 't.sh') - subprocess.check_call(['bash', 't.sh']) - - try: - write_commands_and_test(opts) - except subprocess.CalledProcessError: - print('') - print('====================') - print('Found a problem! See "t.sh" for the commands, and "input.wasm" for the input. Auto-reducing to "reduced.wasm" and "tt.sh"...') - print('====================') - print('') - # first, reduce the fuzz opts: keep removing until we can't - while 1: - reduced = False - for i in range(len(opts)): - # some opts can't be removed, like --flatten --dfo requires flatten - if opts[i] == '--flatten': - if i != len(opts) - 1 and opts[i + 1] in ('--dfo', '--local-cse', '--rereloop'): - continue - shorter = opts[:i] + opts[i + 1:] - try: - write_commands_and_test(shorter) - except subprocess.CalledProcessError: - # great, the shorter one is good as well - opts = shorter - print('reduced opts to ' + ' '.join(opts)) - reduced = True - break - if not reduced: - break - # second, reduce the wasm - # copy a.wasm to a safe place as the reducer will use the commands on new inputs, and the commands work on a.wasm - shutil.copyfile('a.wasm', 'input.wasm') - # add a command to verify the input. this lets the reducer see that it is indeed working on the input correctly - commands = [in_bin('wasm-opt') + ' -all a.wasm'] + get_commands(opts) - write_commands(commands, 'tt.sh') - # reduce the input to something smaller with the same behavior on the script - subprocess.check_call([in_bin('wasm-reduce'), 'input.wasm', '--command=bash tt.sh', '-t', 'a.wasm', '-w', 'reduced.wasm']) - print('Finished reduction. See "tt.sh" and "reduced.wasm".') - sys.exit(1) - print('') - - # created a second wasm for handlers that want to look at pairs. - run([in_bin('wasm-opt'), 'a.wasm', '-o', 'b.wasm'] + opts + FUZZ_OPTS + FEATURE_OPTS) - wasm_size = os.stat('b.wasm').st_size - bytes += wasm_size - print('post wasm size:', wasm_size) - - for testcase_handler in testcase_handlers: - if testcase_handler.can_run_on_feature_opts(FEATURE_OPTS): - if not hasattr(testcase_handler, 'get_commands'): - print('running testcase handler:', testcase_handler.__class__.__name__) - # let the testcase handler handle this testcase however it wants. in this case we give it - # the input and both wasms. - testcase_handler.handle_pair(input=random_input, before_wasm='a.wasm', after_wasm='b.wasm', opts=opts + FUZZ_OPTS + FEATURE_OPTS) - print('') - - return bytes + randomize_pass_debug() + randomize_feature_opts() + + run([in_bin('wasm-opt'), random_input, '-ttf', '-o', 'a.wasm'] + FUZZ_OPTS + FEATURE_OPTS) + wasm_size = os.stat('a.wasm').st_size + bytes = wasm_size + print('pre wasm size:', wasm_size) + + # first, run all handlers that use get_commands(). those don't need the second wasm in the + # pair, since they all they do is return their commands, and expect us to run them, and + # those commands do the actual testing, by operating on the original input wasm file. by + # fuzzing the get_commands() ones first we can find bugs in creating the second wasm (that + # has the opts run on it) before we try to create it later down for the passes that + # expect to get it as one of their inputs. + for testcase_handler in testcase_handlers: + if testcase_handler.can_run_on_feature_opts(FEATURE_OPTS): + if hasattr(testcase_handler, 'get_commands'): + print('running testcase handler:', testcase_handler.__class__.__name__) + # if the testcase handler supports giving us a list of commands, then we can get those commands + # and use them to do useful things like automatic reduction. in this case we give it the input + # wasm plus opts and a random seed (if it needs any internal randomness; we want to have the same + # value there if we reduce). + random_seed = random.random() + + # gets commands from the handler, for a given set of optimizations. this is all the commands + # needed to run the testing that that handler wants to do. + def get_commands(opts): + return testcase_handler.get_commands(wasm='a.wasm', opts=opts + FUZZ_OPTS + FEATURE_OPTS, random_seed=random_seed) + + def write_commands_and_test(opts): + commands = get_commands(opts) + write_commands(commands, 't.sh') + subprocess.check_call(['bash', 't.sh']) + + try: + write_commands_and_test(opts) + except subprocess.CalledProcessError: + print('') + print('====================') + print('Found a problem! See "t.sh" for the commands, and "input.wasm" for the input. Auto-reducing to "reduced.wasm" and "tt.sh"...') + print('====================') + print('') + # first, reduce the fuzz opts: keep removing until we can't + while 1: + reduced = False + for i in range(len(opts)): + # some opts can't be removed, like --flatten --dfo requires flatten + if opts[i] == '--flatten': + if i != len(opts) - 1 and opts[i + 1] in ('--dfo', '--local-cse', '--rereloop'): + continue + shorter = opts[:i] + opts[i + 1:] + try: + write_commands_and_test(shorter) + except subprocess.CalledProcessError: + # great, the shorter one is good as well + opts = shorter + print('reduced opts to ' + ' '.join(opts)) + reduced = True + break + if not reduced: + break + # second, reduce the wasm + # copy a.wasm to a safe place as the reducer will use the commands on new inputs, and the commands work on a.wasm + shutil.copyfile('a.wasm', 'input.wasm') + # add a command to verify the input. this lets the reducer see that it is indeed working on the input correctly + commands = [in_bin('wasm-opt') + ' -all a.wasm'] + get_commands(opts) + write_commands(commands, 'tt.sh') + # reduce the input to something smaller with the same behavior on the script + subprocess.check_call([in_bin('wasm-reduce'), 'input.wasm', '--command=bash tt.sh', '-t', 'a.wasm', '-w', 'reduced.wasm']) + print('Finished reduction. See "tt.sh" and "reduced.wasm".') + sys.exit(1) + print('') + + # created a second wasm for handlers that want to look at pairs. + run([in_bin('wasm-opt'), 'a.wasm', '-o', 'b.wasm'] + opts + FUZZ_OPTS + FEATURE_OPTS) + wasm_size = os.stat('b.wasm').st_size + bytes += wasm_size + print('post wasm size:', wasm_size) + + for testcase_handler in testcase_handlers: + if testcase_handler.can_run_on_feature_opts(FEATURE_OPTS): + if not hasattr(testcase_handler, 'get_commands'): + print('running testcase handler:', testcase_handler.__class__.__name__) + # let the testcase handler handle this testcase however it wants. in this case we give it + # the input and both wasms. + testcase_handler.handle_pair(input=random_input, before_wasm='a.wasm', after_wasm='b.wasm', opts=opts + FUZZ_OPTS + FEATURE_OPTS) + print('') + + return bytes def write_commands(commands, filename): - with open(filename, 'w') as f: - f.write('set -e\n') - for command in commands: - f.write('echo "%s"\n' % command) - pre = 'BINARYEN_PASS_DEBUG=%s ' % (os.environ.get('BINARYEN_PASS_DEBUG') or '0') - f.write(pre + command + ' &> /dev/null\n') - f.write('echo "ok"\n') + with open(filename, 'w') as f: + f.write('set -e\n') + for command in commands: + f.write('echo "%s"\n' % command) + pre = 'BINARYEN_PASS_DEBUG=%s ' % (os.environ.get('BINARYEN_PASS_DEBUG') or '0') + f.write(pre + command + ' &> /dev/null\n') + f.write('echo "ok"\n') # main opt_choices = [ - [], - ['-O1'], ['-O2'], ['-O3'], ['-O4'], ['-Os'], ['-Oz'], - ["--coalesce-locals"], - # XXX slow, non-default ["--coalesce-locals-learning"], - ["--code-pushing"], - ["--code-folding"], - ["--const-hoisting"], - ["--dae"], - ["--dae-optimizing"], - ["--dce"], - ["--directize"], - ["--flatten", "--dfo"], - ["--duplicate-function-elimination"], - ["--flatten"], - # ["--fpcast-emu"], # removes indirect call failures as it makes them go through regardless of type - ["--inlining"], - ["--inlining-optimizing"], - ["--flatten", "--local-cse"], - ["--generate-stack-ir"], - ["--licm"], - ["--memory-packing"], - ["--merge-blocks"], - ['--merge-locals'], - ["--optimize-instructions"], - ["--optimize-stack-ir"], - ["--generate-stack-ir", "--optimize-stack-ir"], - ["--pick-load-signs"], - ["--precompute"], - ["--precompute-propagate"], - ["--print"], - ["--remove-unused-brs"], - ["--remove-unused-nonfunction-module-elements"], - ["--remove-unused-module-elements"], - ["--remove-unused-names"], - ["--reorder-functions"], - ["--reorder-locals"], - ["--flatten", "--rereloop"], - ["--rse"], - ["--simplify-locals"], - ["--simplify-locals-nonesting"], - ["--simplify-locals-nostructure"], - ["--simplify-locals-notee"], - ["--simplify-locals-notee-nostructure"], - ["--ssa"], - ["--vacuum"], + [], + ['-O1'], ['-O2'], ['-O3'], ['-O4'], ['-Os'], ['-Oz'], + ["--coalesce-locals"], + # XXX slow, non-default ["--coalesce-locals-learning"], + ["--code-pushing"], + ["--code-folding"], + ["--const-hoisting"], + ["--dae"], + ["--dae-optimizing"], + ["--dce"], + ["--directize"], + ["--flatten", "--dfo"], + ["--duplicate-function-elimination"], + ["--flatten"], + # ["--fpcast-emu"], # removes indirect call failures as it makes them go through regardless of type + ["--inlining"], + ["--inlining-optimizing"], + ["--flatten", "--local-cse"], + ["--generate-stack-ir"], + ["--licm"], + ["--memory-packing"], + ["--merge-blocks"], + ['--merge-locals'], + ["--optimize-instructions"], + ["--optimize-stack-ir"], + ["--generate-stack-ir", "--optimize-stack-ir"], + ["--pick-load-signs"], + ["--precompute"], + ["--precompute-propagate"], + ["--print"], + ["--remove-unused-brs"], + ["--remove-unused-nonfunction-module-elements"], + ["--remove-unused-module-elements"], + ["--remove-unused-names"], + ["--reorder-functions"], + ["--reorder-locals"], + ["--flatten", "--rereloop"], + ["--rse"], + ["--simplify-locals"], + ["--simplify-locals-nonesting"], + ["--simplify-locals-nostructure"], + ["--simplify-locals-notee"], + ["--simplify-locals-notee-nostructure"], + ["--ssa"], + ["--vacuum"], ] def get_multiple_opt_choices(): - ret = [] - # core opts - while 1: - ret += random.choice(opt_choices) - if len(ret) > 20 or random.random() < 0.3: - break - # modifiers (if not already implied by a -O? option) - if '-O' not in str(ret): - if random.random() < 0.5: - ret += ['--optimize-level=' + str(random.randint(0, 3))] - if random.random() < 0.5: - ret += ['--shrink-level=' + str(random.randint(0, 3))] - return ret + ret = [] + # core opts + while 1: + ret += random.choice(opt_choices) + if len(ret) > 20 or random.random() < 0.3: + break + # modifiers (if not already implied by a -O? option) + if '-O' not in str(ret): + if random.random() < 0.5: + ret += ['--optimize-level=' + str(random.randint(0, 3))] + if random.random() < 0.5: + ret += ['--shrink-level=' + str(random.randint(0, 3))] + return ret # main if not NANS: - FUZZ_OPTS += ['--no-fuzz-nans'] + FUZZ_OPTS += ['--no-fuzz-nans'] # possible feature options that are sometimes passed to the tools. this # contains the list of all possible feature flags we can disable (after @@ -535,21 +535,21 @@ POSSIBLE_FEATURE_OPTS = run([in_bin('wasm-opt'), '--print-features', '-all', in_ print('POSSIBLE_FEATURE_OPTS:', POSSIBLE_FEATURE_OPTS) if __name__ == '__main__': - print('checking infinite random inputs') - random.seed(time.time() * os.getpid()) - temp = 'input.dat' - counter = 0 - bytes = 0 # wasm bytes tested - start_time = time.time() - while True: - counter += 1 - f = open(temp, 'w') - size = random_size() - print('') - print('ITERATION:', counter, 'size:', size, 'speed:', counter / (time.time() - start_time), 'iters/sec, ', bytes / (time.time() - start_time), 'bytes/sec\n') - for x in range(size): - f.write(chr(random.randint(0, 255))) - f.close() - opts = get_multiple_opt_choices() - print('opts:', ' '.join(opts)) - bytes += test_one('input.dat', opts) + print('checking infinite random inputs') + random.seed(time.time() * os.getpid()) + temp = 'input.dat' + counter = 0 + bytes = 0 # wasm bytes tested + start_time = time.time() + while True: + counter += 1 + f = open(temp, 'w') + size = random_size() + print('') + print('ITERATION:', counter, 'size:', size, 'speed:', counter / (time.time() - start_time), 'iters/sec, ', bytes / (time.time() - start_time), 'bytes/sec\n') + for x in range(size): + f.write(chr(random.randint(0, 255))) + f.close() + opts = get_multiple_opt_choices() + print('opts:', ' '.join(opts)) + bytes += test_one('input.dat', opts) |