diff options
-rwxr-xr-x | scripts/clusterfuzz/run.py | 9 | ||||
-rwxr-xr-x | scripts/fuzz_opt.py | 46 | ||||
-rw-r--r-- | scripts/fuzz_shell.js | 79 | ||||
-rw-r--r-- | src/tools/execution-results.h | 3 | ||||
-rw-r--r-- | src/tools/fuzzing.h | 3 | ||||
-rw-r--r-- | src/tools/fuzzing/fuzzing.cpp | 29 | ||||
-rw-r--r-- | test/lit/exec/fuzzing-api.wast | 19 | ||||
-rw-r--r-- | test/passes/fuzz_metrics_noprint.bin.txt | 52 | ||||
-rw-r--r-- | test/passes/fuzz_metrics_passes_noprint.bin.txt | 53 | ||||
-rw-r--r-- | test/passes/translate-to-fuzz_all-features_metrics_noprint.txt | 83 | ||||
-rw-r--r-- | test/unit/test_cluster_fuzz.py | 22 |
11 files changed, 279 insertions, 119 deletions
diff --git a/scripts/clusterfuzz/run.py b/scripts/clusterfuzz/run.py index 8ac880e0d..2fedb6510 100755 --- a/scripts/clusterfuzz/run.py +++ b/scripts/clusterfuzz/run.py @@ -200,6 +200,15 @@ def get_js_file_contents(i, output_dir): print(f'Created {bytes} wasm bytes') + # Some of the time, fuzz JSPI (similar to fuzz_opt.py, see details there). + if system_random.random() < 0.25: + # Prepend the flag to enable JSPI. + js = 'var JSPI = 1;\n\n' + js + + # Un-comment the async and await keywords. + js = js.replace('/* async */', 'async') + js = js.replace('/* await */', 'await') + return js diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index 87f059058..ca7c9e355 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -232,7 +232,11 @@ def randomize_fuzz_settings(): if random.random() < 0.5: GEN_ARGS += ['--enclose-world'] - print('randomized settings (NaNs, OOB, legalize):', NANS, OOB, LEGALIZE) + # Test JSPI somewhat rarely, as it may be slower. + global JSPI + JSPI = random.random() < 0.25 + + print('randomized settings (NaNs, OOB, legalize, JSPI):', NANS, OOB, LEGALIZE, JSPI) def init_important_initial_contents(): @@ -758,11 +762,39 @@ def run_d8_js(js, args=[], liftoff=True): return run_vm(cmd) -FUZZ_SHELL_JS = in_binaryen('scripts', 'fuzz_shell.js') +# For JSPI, we must customize fuzz_shell.js. We do so the first time we need +# it, and save the filename here. +JSPI_JS_FILE = None + + +def get_fuzz_shell_js(): + js = in_binaryen('scripts', 'fuzz_shell.js') + + if not JSPI: + # Just use the normal fuzz shell script. + return js + + global JSPI_JS_FILE + if JSPI_JS_FILE: + # Use the customized file we've already created. + return JSPI_JS_FILE + + JSPI_JS_FILE = os.path.abspath('jspi_fuzz_shell.js') + with open(JSPI_JS_FILE, 'w') as f: + # Enable JSPI. + f.write('var JSPI = 1;\n\n') + + # Un-comment the async and await keywords. + with open(js) as g: + code = g.read() + code = code.replace('/* async */', 'async') + code = code.replace('/* await */', 'await') + f.write(code) + return JSPI_JS_FILE def run_d8_wasm(wasm, liftoff=True, args=[]): - return run_d8_js(FUZZ_SHELL_JS, [wasm] + args, liftoff=liftoff) + return run_d8_js(get_fuzz_shell_js(), [wasm] + args, liftoff=liftoff) def all_disallowed(features): @@ -850,7 +882,7 @@ class CompareVMs(TestCaseHandler): name = 'd8' def run(self, wasm, extra_d8_flags=[]): - return run_vm([shared.V8, FUZZ_SHELL_JS] + shared.V8_OPTS + get_v8_extra_flags() + extra_d8_flags + ['--', wasm]) + return run_vm([shared.V8, get_fuzz_shell_js()] + shared.V8_OPTS + get_v8_extra_flags() + extra_d8_flags + ['--', wasm]) def can_run(self, wasm): # V8 does not support shared memories when running with @@ -1160,7 +1192,7 @@ class Wasm2JS(TestCaseHandler): compare_between_vms(before, interpreter, 'Wasm2JS (vs interpreter)') def run(self, wasm): - with open(FUZZ_SHELL_JS) as f: + with open(get_fuzz_shell_js()) as f: wrapper = f.read() cmd = [in_bin('wasm2js'), wasm, '--emscripten'] # avoid optimizations if we have nans, as we don't handle them with @@ -1193,6 +1225,10 @@ class Wasm2JS(TestCaseHandler): # specifically for growth here if INITIAL_CONTENTS: return False + # We run in node, which lacks JSPI support, and also we need wasm2js to + # implement wasm suspending using JS async/await. + if JSPI: + return False return all_disallowed(['exception-handling', 'simd', 'threads', 'bulk-memory', 'nontrapping-float-to-int', 'tail-call', 'sign-ext', 'reference-types', 'multivalue', 'gc', 'multimemory', 'memory64']) diff --git a/scripts/fuzz_shell.js b/scripts/fuzz_shell.js index 95176bbe6..653146517 100644 --- a/scripts/fuzz_shell.js +++ b/scripts/fuzz_shell.js @@ -1,3 +1,17 @@ +// This script can be customized by setting the following variables in code that +// runs before this script. +// +// The binary to be run. (If not set, we get the filename from argv and read +// from it.) +var binary; +// A second binary to be linked in and run as well. (Can also be read from +// argv.) +var secondBinary; +// Whether we are fuzzing JSPI. In addition to this being set, the "async" and +// "await" keywords must be taken out of the /* KEYWORD */ comments (which they +// are normally in, so as not to affect normal fuzzing). +var JSPI; + // Shell integration: find argv and set up readBinary(). var argv; var readBinary; @@ -25,9 +39,6 @@ if (typeof process === 'object' && typeof require === 'function') { }; } -// The binary to be run. This may be set already (by code that runs before this -// script), and if not, we get the filename from argv. -var binary; if (!binary) { binary = readBinary(argv[0]); } @@ -43,7 +54,6 @@ if (argv.length > 0 && argv[argv.length - 1].startsWith('exports:')) { // If a second parameter is given, it is a second binary that we will link in // with it. -var secondBinary; if (argv[1]) { secondBinary = readBinary(argv[1]); } @@ -163,9 +173,9 @@ function callFunc(func) { // Calls a given function in a try-catch, swallowing JS exceptions, and return 1 // if we did in fact swallow an exception. Wasm traps are not swallowed (see // details below). -function tryCall(func) { +/* async */ function tryCall(func) { try { - func(); + /* await */ func(); return 0; } catch (e) { // We only want to catch exceptions, not wasm traps: traps should still @@ -243,19 +253,39 @@ var imports = { }, // Export operations. - 'call-export': (index) => { - callFunc(exportList[index].value); + 'call-export': /* async */ (index) => { + /* await */ callFunc(exportList[index].value); }, - 'call-export-catch': (index) => { - return tryCall(() => callFunc(exportList[index].value)); + 'call-export-catch': /* async */ (index) => { + return tryCall(/* async */ () => /* await */ callFunc(exportList[index].value)); }, // Funcref operations. - 'call-ref': (ref) => { - callFunc(ref); + 'call-ref': /* async */ (ref) => { + // This is a direct function reference, and just like an export, it must + // be wrapped for JSPI. + ref = wrapExportForJSPI(ref); + /* await */ callFunc(ref); + }, + 'call-ref-catch': /* async */ (ref) => { + ref = wrapExportForJSPI(ref); + return tryCall(/* async */ () => /* await */ callFunc(ref)); }, - 'call-ref-catch': (ref) => { - return tryCall(() => callFunc(ref)); + + // Sleep a given amount of ms (when JSPI) and return a given id after that. + 'sleep': (ms, id) => { + if (!JSPI) { + return id; + } + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve(id); + }, 0); // TODO: Use the ms in some reasonable, deterministic manner. + // Rather than actually setTimeout on them we could manage + // a queue of pending sleeps manually, and order them based + // on the "ms" (which would not be literal ms, but just + // how many time units to wait). + }); }, }, // Emscripten support. @@ -274,6 +304,22 @@ if (typeof WebAssembly.Tag !== 'undefined') { }; } +// If JSPI is available, wrap the imports and exports. +if (JSPI) { + for (var name of ['sleep', 'call-export', 'call-export-catch', 'call-ref', + 'call-ref-catch']) { + imports['fuzzing-support'][name] = + new WebAssembly.Suspending(imports['fuzzing-support'][name]); + } +} + +function wrapExportForJSPI(value) { + if (JSPI && typeof value === 'function') { + value = WebAssembly.promising(value); + } + return value; +} + // If a second binary will be linked in then set up the imports for // placeholders. Any import like (import "placeholder" "0" (func .. will be // provided by the secondary module, and must be called using an indirection. @@ -312,13 +358,14 @@ function build(binary) { // keep the ability to call anything that was ever exported.) for (var key in instance.exports) { var value = instance.exports[key]; + value = wrapExportForJSPI(value); exports[key] = value; exportList.push({ name: key, value: value }); } } // Run the code by calling exports. -function callExports() { +/* async */ function callExports() { // Call the exports we were told, or if we were not given an explicit list, // call them all. var relevantExports = exportsToCall || exportList; @@ -342,7 +389,7 @@ function callExports() { try { console.log('[fuzz-exec] calling ' + name); - var result = callFunc(value); + var result = /* await */ callFunc(value); if (typeof result !== 'undefined') { console.log('[fuzz-exec] note result: ' + name + ' => ' + printed(result)); } diff --git a/src/tools/execution-results.h b/src/tools/execution-results.h index b8823c3f0..ea822d547 100644 --- a/src/tools/execution-results.h +++ b/src/tools/execution-results.h @@ -128,6 +128,9 @@ public: } catch (const WasmException& e) { return {Literal(int32_t(1))}; } + } else if (import->base == "sleep") { + // Do not actually sleep, just return the id. + return {arguments[1]}; } else { WASM_UNREACHABLE("unknown fuzzer import"); } diff --git a/src/tools/fuzzing.h b/src/tools/fuzzing.h index 78219045c..f76ed62a5 100644 --- a/src/tools/fuzzing.h +++ b/src/tools/fuzzing.h @@ -117,6 +117,7 @@ private: Name callExportCatchImportName; Name callRefImportName; Name callRefCatchImportName; + Name sleepImportName; std::unordered_map<Type, std::vector<Name>> globalsByType; std::unordered_map<Type, std::vector<Name>> mutableGlobalsByType; @@ -238,6 +239,7 @@ private: void addImportCallingSupport(); void addImportThrowingSupport(); void addImportTableSupport(); + void addImportSleepSupport(); void addHashMemorySupport(); // Special expression makers @@ -249,6 +251,7 @@ private: // Call either an export or a ref. We do this from a single function to better // control the frequency of each. Expression* makeImportCallCode(Type type); + Expression* makeImportSleep(Type type); Expression* makeMemoryHashLogging(); // Function creation diff --git a/src/tools/fuzzing/fuzzing.cpp b/src/tools/fuzzing/fuzzing.cpp index a7f5e0d01..2b9286180 100644 --- a/src/tools/fuzzing/fuzzing.cpp +++ b/src/tools/fuzzing/fuzzing.cpp @@ -317,6 +317,7 @@ void TranslateToFuzzReader::build() { } addImportLoggingSupport(); addImportCallingSupport(); + addImportSleepSupport(); modifyInitialFunctions(); // keep adding functions until we run out of input while (!random.finished()) { @@ -909,6 +910,24 @@ void TranslateToFuzzReader::addImportTableSupport() { } } +void TranslateToFuzzReader::addImportSleepSupport() { + if (!oneIn(4)) { + // Fuzz this somewhat rarely, as it may be slow. + return; + } + + // An import that sleeps for a given number of milliseconds, and also receives + // an integer id. It returns that integer id (useful for tracking separate + // sleeps). + sleepImportName = Names::getValidFunctionName(wasm, "sleep"); + auto func = std::make_unique<Function>(); + func->name = sleepImportName; + func->module = "fuzzing-support"; + func->base = "sleep"; + func->type = Signature({Type::i32, Type::i32}, Type::i32); + wasm.addFunction(std::move(func)); +} + void TranslateToFuzzReader::addHashMemorySupport() { // Add memory hasher helper (for the hash, see hash.h). The function looks // like: @@ -1090,6 +1109,13 @@ Expression* TranslateToFuzzReader::makeImportCallCode(Type type) { return builder.makeCall(exportTarget, {index}, type); } +Expression* TranslateToFuzzReader::makeImportSleep(Type type) { + // Sleep for some ms, and return a given id. + auto* ms = make(Type::i32); + auto id = make(Type::i32); + return builder.makeCall(sleepImportName, {ms, id}, Type::i32); +} + Expression* TranslateToFuzzReader::makeMemoryHashLogging() { auto* hash = builder.makeCall(std::string("hashMemory"), {}, Type::i32); return builder.makeCall(logImportNames[Type::i32], {hash}, Type::none); @@ -1768,6 +1794,9 @@ Expression* TranslateToFuzzReader::_makeConcrete(Type type) { if (callExportCatchImportName || callRefCatchImportName) { options.add(FeatureSet::MVP, &Self::makeImportCallCode); } + if (sleepImportName) { + options.add(FeatureSet::MVP, &Self::makeImportSleep); + } options.add(FeatureSet::ReferenceTypes, &Self::makeRefIsNull); options.add(FeatureSet::ReferenceTypes | FeatureSet::GC, &Self::makeRefEq, diff --git a/test/lit/exec/fuzzing-api.wast b/test/lit/exec/fuzzing-api.wast index 7c975cb75..8e251b2ed 100644 --- a/test/lit/exec/fuzzing-api.wast +++ b/test/lit/exec/fuzzing-api.wast @@ -19,6 +19,8 @@ (import "fuzzing-support" "call-ref" (func $call.ref (param funcref))) (import "fuzzing-support" "call-ref-catch" (func $call.ref.catch (param funcref) (result i32))) + (import "fuzzing-support" "sleep" (func $sleep (param i32 i32) (result i32))) + (table $table 10 20 funcref) ;; Note that the exported table appears first here, but in the binary and in @@ -284,7 +286,6 @@ ;; CHECK: [fuzz-exec] calling ref.calling.trap ;; CHECK-NEXT: [trap unreachable] - ;; CHECK-NEXT: warning: no passes specified, not doing any work (func $ref.calling.trap (export "ref.calling.trap") ;; We try to catch an exception here, but the target function traps, which is ;; not something we can catch. We will trap here, and not log at all. @@ -294,6 +295,18 @@ ) ) ) + + ;; CHECK: [fuzz-exec] calling do-sleep + ;; CHECK-NEXT: [fuzz-exec] note result: do-sleep => 42 + ;; CHECK-NEXT: warning: no passes specified, not doing any work + (func $do-sleep (export "do-sleep") (result i32) + (call $sleep + ;; A ridiculous amount of ms, but in the interpreter it is ignored anyhow. + (i32.const -1) + ;; An id, that is returned back to us. + (i32.const 42) + ) + ) ) ;; CHECK: [fuzz-exec] calling logging ;; CHECK-NEXT: [LoggingExternalInterface logging 42] @@ -354,6 +367,10 @@ ;; CHECK: [fuzz-exec] calling ref.calling.trap ;; CHECK-NEXT: [trap unreachable] + +;; CHECK: [fuzz-exec] calling do-sleep +;; CHECK-NEXT: [fuzz-exec] note result: do-sleep => 42 +;; CHECK-NEXT: [fuzz-exec] comparing do-sleep ;; CHECK-NEXT: [fuzz-exec] comparing export.calling ;; CHECK-NEXT: [fuzz-exec] comparing export.calling.catching ;; CHECK-NEXT: [fuzz-exec] comparing logging diff --git a/test/passes/fuzz_metrics_noprint.bin.txt b/test/passes/fuzz_metrics_noprint.bin.txt index a2f996bcb..bf7e517da 100644 --- a/test/passes/fuzz_metrics_noprint.bin.txt +++ b/test/passes/fuzz_metrics_noprint.bin.txt @@ -1,35 +1,35 @@ Metrics total - [exports] : 49 - [funcs] : 74 + [exports] : 46 + [funcs] : 68 [globals] : 18 [imports] : 4 [memories] : 1 [memory-data] : 24 - [table-data] : 19 + [table-data] : 22 [tables] : 1 [tags] : 0 - [total] : 5695 - [vars] : 227 - Binary : 430 - Block : 976 - Break : 188 - Call : 272 - CallIndirect : 20 - Const : 876 - Drop : 99 - GlobalGet : 504 - GlobalSet : 373 - If : 299 - Load : 111 - LocalGet : 365 - LocalSet : 299 - Loop : 108 - Nop : 58 - RefFunc : 19 - Return : 74 - Select : 40 - Store : 34 + [total] : 9465 + [vars] : 215 + Binary : 671 + Block : 1531 + Break : 370 + Call : 366 + CallIndirect : 67 + Const : 1478 + Drop : 111 + GlobalGet : 766 + GlobalSet : 558 + If : 514 + Load : 173 + LocalGet : 729 + LocalSet : 550 + Loop : 202 + Nop : 133 + RefFunc : 22 + Return : 99 + Select : 84 + Store : 83 Switch : 2 - Unary : 365 - Unreachable : 183 + Unary : 682 + Unreachable : 274 diff --git a/test/passes/fuzz_metrics_passes_noprint.bin.txt b/test/passes/fuzz_metrics_passes_noprint.bin.txt index c3881104d..814fdb132 100644 --- a/test/passes/fuzz_metrics_passes_noprint.bin.txt +++ b/test/passes/fuzz_metrics_passes_noprint.bin.txt @@ -1,34 +1,35 @@ Metrics total - [exports] : 30 - [funcs] : 47 + [exports] : 43 + [funcs] : 56 [globals] : 17 [imports] : 4 [memories] : 1 [memory-data] : 11 - [table-data] : 18 + [table-data] : 16 [tables] : 1 [tags] : 0 - [total] : 4738 - [vars] : 133 - Binary : 338 - Block : 781 - Break : 122 - Call : 249 - CallIndirect : 27 - Const : 780 - Drop : 105 - GlobalGet : 378 - GlobalSet : 288 - If : 216 - Load : 79 - LocalGet : 396 - LocalSet : 252 - Loop : 89 - Nop : 43 - RefFunc : 18 - Return : 70 - Select : 37 - Store : 36 - Unary : 294 - Unreachable : 140 + [total] : 10611 + [vars] : 184 + Binary : 754 + Block : 1699 + Break : 397 + Call : 325 + CallIndirect : 112 + Const : 1783 + Drop : 101 + GlobalGet : 869 + GlobalSet : 657 + If : 549 + Load : 195 + LocalGet : 893 + LocalSet : 609 + Loop : 251 + Nop : 123 + RefFunc : 16 + Return : 78 + Select : 74 + Store : 84 + Switch : 3 + Unary : 730 + Unreachable : 309 diff --git a/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt b/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt index e2e9b8053..3c2b7bfa2 100644 --- a/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt +++ b/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt @@ -1,57 +1,52 @@ Metrics total - [exports] : 6 - [funcs] : 6 + [exports] : 3 + [funcs] : 3 [globals] : 4 [imports] : 8 [memories] : 1 [memory-data] : 112 - [table-data] : 1 + [table-data] : 0 [tables] : 1 [tags] : 1 - [total] : 592 - [vars] : 38 - ArrayGet : 2 - ArrayLen : 2 - ArrayNew : 6 - ArrayNewFixed : 4 - ArraySet : 1 + [total] : 630 + [vars] : 23 + ArrayNewFixed : 3 AtomicCmpxchg : 1 AtomicFence : 1 - AtomicRMW : 2 - Binary : 81 - Block : 62 - BrOn : 1 - Break : 11 - Call : 13 - CallIndirect : 2 - Const : 123 - Drop : 2 - GlobalGet : 22 - GlobalSet : 22 - If : 17 - Load : 25 - LocalGet : 63 - LocalSet : 35 - Loop : 6 - Nop : 8 - Pop : 3 - RefAs : 1 - RefEq : 1 - RefFunc : 6 - RefNull : 3 - Return : 4 - SIMDExtract : 2 - Select : 2 - StringConst : 5 + AtomicNotify : 1 + Binary : 63 + Block : 60 + BrOn : 3 + Break : 8 + Call : 4 + CallRef : 3 + Const : 129 + DataDrop : 1 + Drop : 8 + GlobalGet : 21 + GlobalSet : 20 + I31Get : 1 + If : 12 + Load : 17 + LocalGet : 74 + LocalSet : 52 + Loop : 8 + MemoryFill : 1 + Nop : 4 + RefAs : 19 + RefFunc : 26 + RefI31 : 1 + RefIsNull : 1 + RefNull : 11 + Return : 2 + Select : 5 StringEncode : 1 - StringEq : 2 - StringMeasure : 1 - StringWTF16Get : 2 - StructNew : 9 - Try : 3 - TryTable : 3 + StructGet : 3 + StructNew : 26 + Try : 1 + TryTable : 6 TupleExtract : 1 TupleMake : 2 - Unary : 18 - Unreachable : 11 + Unary : 20 + Unreachable : 10 diff --git a/test/unit/test_cluster_fuzz.py b/test/unit/test_cluster_fuzz.py index 56250d46a..8f1d18104 100644 --- a/test/unit/test_cluster_fuzz.py +++ b/test/unit/test_cluster_fuzz.py @@ -274,10 +274,11 @@ class ClusterFuzz(utils.BinaryenTestCase): print() # To check for interesting JS file contents, we'll note how many times - # we build and run the wasm. + # we build and run the wasm, and other things like JSPI. seen_builds = [] seen_calls = [] seen_second_builds = [] + seen_JSPIs = [] for i in range(1, N + 1): fuzz_file = os.path.join(temp_dir.name, f'fuzz-binaryen-{i}.js') @@ -287,6 +288,17 @@ class ClusterFuzz(utils.BinaryenTestCase): seen_calls.append(js.count('callExports();')) seen_second_builds.append(js.count('build(secondBinary);')) + # If JSPI is enabled, the async and await keywords should be + # enabled (uncommented). + if 'JSPI = 1' in js: + seen_JSPIs.append(1) + assert '/* async */' not in js + assert '/* await */' not in js + else: + seen_JSPIs.append(0) + assert '/* async */' in js + assert '/* await */' in js + # There is always one build and one call (those are in the default # fuzz_shell.js), and we add a couple of operations, each with equal # probability to be a build or a call, so over the 100 testcases here we @@ -323,6 +335,14 @@ class ClusterFuzz(utils.BinaryenTestCase): print() + # JSPI is done 1/4 of the time or so. + print('JSPIs are distributed as ~ mean 0.25') + print(f'mean JSPIs: {statistics.mean(seen_JSPIs)}') + self.assertEqual(min(seen_JSPIs), 0) + self.assertEqual(max(seen_JSPIs), 1) + + print() + # "zzz" in test name so that this runs last. If it runs first, it can be # confusing as it appears next to the logging of which bundle we use (see # setUpClass). |