diff options
| -rw-r--r-- | scripts/fuzz_opt.py | 226 |
1 files changed, 226 insertions, 0 deletions
diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py new file mode 100644 index 000000000..3330000c9 --- /dev/null +++ b/scripts/fuzz_opt.py @@ -0,0 +1,226 @@ +''' +Runs random passes and options on random inputs, using wasm-opt. + +Can be configured to run just wasm-opt itself (using --fuzz-exec) +or also run VMs on it. + +For afl-fuzz integration, you probably don't want this, and can use +something like + +BINARYEN_CORES=1 BINARYEN_PASS_DEBUG=1 afl-fuzz -i afl-testcases/ -o afl-findings/ -m 100 -d -- bin/wasm-opt -ttf --fuzz-exec --Os @@ + +(that is on a fixed set of arguments to wasm-opt, though - this +script covers different options being passed) +''' + +import os +import sys +import difflib +import subprocess +import random +import shutil +import time + +# parameters + +LOG_LIMIT = 125 +INPUT_SIZE_LIMIT = 250 * 1024 + + +def random_size(): + return random.randint(1, INPUT_SIZE_LIMIT) + + +def run(cmd): + print(' '.join(cmd)[:LOG_LIMIT]) + return subprocess.check_output(cmd) + + +def run_unchecked(cmd): + print(' '.join(cmd)[:LOG_LIMIT]) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0] + + +def randomize_pass_debug(): + if random.random() < 0.125: + print('[pass-debug]') + os.environ['BINARYEN_PASS_DEBUG'] = '1' + else: + os.environ['BINARYEN_PASS_DEBUG'] = '0' + del os.environ['BINARYEN_PASS_DEBUG'] + + +def test_one(infile, opts): + def compare(x, y, comment): + if x != y: + message = ''.join([a.rstrip() + '\n' for a in difflib.unified_diff(x.split('\n'), y.split('\n'), fromfile='expected', tofile='actual')]) + raise Exception(str(comment) + ": Expected to have '%s' == '%s', diff:\n\n%s" % ( + x, y, + message + )) + + def run_vms(prefix): + def fix_output(out): + # exceptions may differ when optimizing, but an exception should occur. so ignore their types + # also js engines print them out slightly differently + return '\n'.join(map(lambda x: ' *exception*' if 'exception' in x else x, out.split('\n'))) + + # normalize different vm output + # also the binaryen optimizer can reorder traps (but not remove them), so + # it really just matters if you trap, not how you trap + return out.replace('unreachable executed', 'unreachable') \ + .replace('integer result unrepresentable', 'integer overflow') \ + .replace('invalid conversion to integer', 'integer overflow') \ + .replace('memory access out of bounds', 'index out of bounds') \ + .replace('integer divide by zero', 'divide by zero') \ + .replace('integer remainder by zero', 'remainder by zero') \ + .replace('remainder by zero', 'divide by zero') \ + .replace('divide result unrepresentable', 'integer overflow') \ + .replace('divide by zero', 'integer overflow') \ + .replace('index out of bounds', 'integer overflow') \ + .replace('out of bounds memory access', 'integer overflow') + + def fix_spec_output(out): + out = fix_output(out) + # spec shows a pointer when it traps, remove that + out = '\n'.join(map(lambda x: x if 'runtime trap' not in x else x[x.find('runtime trap'):], out.split('\n'))) + # https://github.com/WebAssembly/spec/issues/543 , float consts are messed up + out = '\n'.join(map(lambda x: x if 'f32' not in x and 'f64' not in x else '', out.split('\n'))) + return out + + results = [] + # append to this list to add results from VMs + # results += [fix_output(run([os.path.expanduser('d8'), '--', prefix + 'js', prefix + 'wasm']))] + # spec has no mechanism to not halt on a trap. so we just check until the first trap, basically + # run(['../spec/interpreter/wasm', prefix + 'wasm']) + # results += [fix_spec_output(run_unchecked(['../spec/interpreter/wasm', prefix + 'wasm', '-e', open(prefix + 'wat').read()]))] + + if len(results) == 0: + results = [0] + + first = results[0] + for i in range(len(results)): + compare(first, results[i], 'comparing between vms at ' + str(i)) + + return results + + randomize_pass_debug() + + bytes = 0 + + # fuzz vms + # gather VM outputs on input file + run(['bin/wasm-opt', infile, '-ttf', '--emit-js-wrapper=a.js', '--emit-spec-wrapper=a.wat', '-o', 'a.wasm']) + wasm_size = os.stat('a.wasm').st_size + bytes += wasm_size + print('pre js size :', os.stat('a.js').st_size, ' wasm size:', wasm_size) + before = run_vms('a.') + print('----------------') + # gather VM outputs on processed file + run(['bin/wasm-opt', 'a.wasm', '-o', 'b.wasm'] + opts) + wasm_size = os.stat('b.wasm').st_size + bytes += wasm_size + print('post js size:', os.stat('a.js').st_size, ' wasm size:', wasm_size) + shutil.copyfile('a.js', 'b.js') + after = run_vms('b.') + for i in range(len(before)): + compare(before[i], after[i], 'comparing between builds at ' + str(i)) + # fuzz binaryen interpreter itself. separate invocation so result is easily fuzzable + run(['bin/wasm-opt', 'a.wasm', '--fuzz-exec', '--fuzz-binary'] + opts) + + return bytes + + +# main + +opt_choices = [ + [], + ['-O1'], ['-O2'], ['-O3'], ['-O4'], ['-Os'], ['-Oz'], + ["--coalesce-locals"], + # XXX slow, non-default ["--coalesce-locals-learning"], + ["--code-pushing"], + ["--code-folding"], + ["--const-hoisting"], + ["--dae"], + ["--dae-optimizing"], + ["--dce"], + ["--flatten", "--dfo"], + ["--duplicate-function-elimination"], + ["--flatten"], + # ["--fpcast-emu"], # removes indirect call failures as it makes them go through regardless of type + ["--inlining"], + ["--inlining-optimizing"], + ["--flatten", "--local-cse"], + ["--generate-stack-ir"], + ["--licm"], + ["--memory-packing"], + ["--merge-blocks"], + ['--merge-locals'], + ["--optimize-instructions"], + ["--optimize-stack-ir"], + ["--generate-stack-ir", "--optimize-stack-ir"], + ["--pick-load-signs"], + ["--precompute"], + ["--precompute-propagate"], + ["--remove-unused-brs"], + ["--remove-unused-nonfunction-module-elements"], + ["--remove-unused-module-elements"], + ["--remove-unused-names"], + ["--reorder-functions"], + ["--reorder-locals"], + ["--flatten", "--rereloop"], + ["--rse"], + ["--simplify-locals"], + ["--simplify-locals-nonesting"], + ["--simplify-locals-nostructure"], + ["--simplify-locals-notee"], + ["--simplify-locals-notee-nostructure"], + ["--ssa"], + ["--vacuum"], +] + + +def get_multiple_opt_choices(): + ret = [] + # core opts + while 1: + ret += random.choice(opt_choices) + if len(ret) > 20 or random.random() < 0.3: + break + # modifiers (if not already implied by a -O? option) + if '-O' not in str(ret): + if random.random() < 0.5: + ret += ['--optimize-level=' + str(random.randint(0, 3))] + if random.random() < 0.5: + ret += ['--shrink-level=' + str(random.randint(0, 3))] + return ret + + +# main + +if len(sys.argv) >= 2: + print('checking given input') + if len(sys.argv) >= 3: + test_one(sys.argv[1], sys.argv[2:]) + else: + for opts in opt_choices: + print(opts) + test_one(sys.argv[1], opts) +else: + print('checking infinite random inputs') + random.seed(time.time()) + temp = 'input.dat' + counter = 0 + bytes = 0 # wasm bytes tested + start_time = time.time() + while True: + counter += 1 + f = open(temp, 'w') + size = random_size() + print('\nITERATION:', counter, 'size:', size, 'speed:', counter / (time.time() - start_time), 'iters/sec, ', bytes / (time.time() - start_time), 'bytes/sec\n') + for x in range(size): + f.write(chr(random.randint(0, 255))) + f.close() + opts = get_multiple_opt_choices() + print('opts:', ' '.join(opts)) + bytes += test_one('input.dat', opts) |