Fuzzing: Append more JS operations in run.py (#7098)

The main fuzz_shell.js code builds and runs the given wasm. After the refactoring
in #7096, it is simple to append to that file and add more build and run operations,
adding more variety to the code, including cross-module interactions. Add logic
to run.py to do that for ClusterFuzz.

To test this, add a node test that builds a module with internal state that can
actually show which module is being executed. The test appends a build+run
operation, whose output prove that we are calling from the first module to the
second and vice versa.

Also add a ClusterFuzz test for run.py that verifies that we add a variety of
build/run operations.

3 files changed, 215 insertions, 1 deletions

diff --git a/scripts/clusterfuzz/run.py b/scripts/clusterfuzz/run.py
index efddfc2d4..4b5e67fde 100755
--- a/scripts/clusterfuzz/run.py
+++ b/scripts/clusterfuzz/run.py
@@ -25,10 +25,12 @@ bundle_clusterfuzz.py.
 
 import os
 import getopt
+import math
 import random
 import subprocess
 import sys
 
+
 # The V8 flags we put in the "fuzzer flags" files, which tell ClusterFuzz how to
 # run V8. By default we apply all staging flags.
 FUZZER_FLAGS_FILE_CONTENTS = '--wasm-staging'
@@ -39,6 +41,12 @@ FUZZER_FLAGS_FILE_CONTENTS = '--wasm-staging'
 # processes per file), which is less of an issue on ClusterFuzz.
 MAX_RANDOM_SIZE = 15 * 1024
 
+# Max and median amount of extra JS operations we append, like extra compiles or
+# runs of the wasm. We allow a high max, but the median is far lower, so that
+# typical testcases are not long-running.
+MAX_EXTRA_JS_OPERATIONS = 40
+MEDIAN_EXTRA_JS_OPERATIONS = 2
+
 # The prefix for fuzz files.
 FUZZ_FILENAME_PREFIX = 'fuzz-'
 
@@ -80,6 +88,11 @@ def get_file_name(prefix, index):
     return f'{prefix}{FUZZER_NAME_PREFIX}{index}.js'
 
 
+# We should only use the system's random number generation, which is the best.
+# (We also use urandom below, which uses this under the hood.)
+system_random = random.SystemRandom()
+
+
 # Returns the contents of a .js fuzz file, given particular wasm contents that
 # we want to be executed.
 def get_js_file_contents(wasm_contents):
@@ -91,6 +104,35 @@ def get_js_file_contents(wasm_contents):
     # mechanism where the wasm file's name is provided in argv).
     wasm_contents = ','.join([str(c) for c in wasm_contents])
     js = f'var binary = new Uint8Array([{wasm_contents}]);\n\n' + js
+
+    # The default JS builds and runs the wasm. Append some random additional
+    # operations as well, as more compiles and executions can find things. To
+    # approximate a number in the range [0, MAX_EXTRA_JS_OPERATIONS) but with a
+    # median of MEDIAN_EXTRA_JS_OPERATIONS, start in the range [0, 1) and then
+    # raise it to the proper power, as multiplying by itself keeps the range
+    # unchanged, but lowers the median. Specifically, the median begins at 0.5,
+    # so
+    #
+    #   0.5^power = MEDIAN_EXTRA_JS_OPERATIONS / MAX_EXTRA_JS_OPERATIONS
+    #
+    # is what we want, and if we take log2 of each side, gives us
+    #
+    #   power =  log2(MEDIAN_EXTRA_JS_OPERATIONS / MAX_EXTRA_JS_OPERATIONS) / log2(0.5)
+    #         = -log2(MEDIAN_EXTRA_JS_OPERATIONS / MAX_EXTRA_JS_OPERATIONS)
+    power = -math.log2(float(MEDIAN_EXTRA_JS_OPERATIONS) / MAX_EXTRA_JS_OPERATIONS)
+    x = system_random.random()
+    x = math.pow(x, power)
+    num = math.floor(x * MAX_EXTRA_JS_OPERATIONS)
+    assert num >= 0 and num <= MAX_EXTRA_JS_OPERATIONS
+    for i in range(num):
+        js += system_random.choice([
+            # Compile and link the wasm again. Each link adds more to the total
+            # exports that we can call.
+            'build(binary);\n',
+            # Run all the exports we've accumulated.
+            'callExports();\n',
+        ])
+
     return js
 
 
@@ -115,7 +157,7 @@ def main(argv):
         # detects as invalid). Just try again in such a case.
         for attempt in range(0, 100):
             # Generate random data.
-            random_size = random.SystemRandom().randint(1, MAX_RANDOM_SIZE)
+            random_size = system_random.randint(1, MAX_RANDOM_SIZE)
             with open(input_data_file_path, 'wb') as file:
                 file.write(os.urandom(random_size))
 
diff --git a/test/lit/node/fuzz_shell_append.wast b/test/lit/node/fuzz_shell_append.wast
new file mode 100644
index 000000000..4aa61d536
--- /dev/null
+++ b/test/lit/node/fuzz_shell_append.wast
@@ -0,0 +1,135 @@
+;; Test that appending more build and run operations, as the ClusterFuzz run.py
+;; does, works properly.
+
+(module
+  (import "fuzzing-support" "log-i32" (func $log (param i32)))
+  (import "fuzzing-support" "call-export-catch" (func $call.export.catch (param i32) (result i32)))
+
+  (global $errors (mut i32)
+    (i32.const 0)
+  )
+
+  (func $errors (export "errors")
+    ;; Log the number of errors we've seen.
+    (call $log
+      (global.get $errors)
+    )
+  )
+
+  (func $do-call (param $x i32)
+    ;; Given an index $x, call the export of that index, and note an error if
+    ;; we see one.
+    (if
+      (call $call.export.catch
+        (local.get $x)
+      )
+      (then
+        ;; Log that we errored right now, and then increment the total.
+        (call $log
+          (i32.const -1)
+        )
+        (global.set $errors
+          (i32.add
+            (global.get $errors)
+            (i32.const 1)
+          )
+        )
+      )
+    )
+    ;; Log the total number of errors so far.
+    (call $log
+      (global.get $errors)
+    )
+  )
+
+  (func $call-0 (export "call0")
+    ;; This calls "errors".
+    (call $do-call
+      (i32.const 0)
+    )
+  )
+
+  (func $call-3 (export "call3")
+    ;; The first time we try this, there is no export at index 3, since we just
+    ;; have ["errors", "call0", "call3"]. After we build the module a second
+    ;; time, we will have "errors" from the second module there.
+    (call $do-call
+      (i32.const 3)
+    )
+  )
+)
+
+;; Run normally.
+;;
+;; RUN: wasm-opt %s -o %t.wasm -q
+;; RUN: node %S/../../../scripts/fuzz_shell.js %t.wasm | filecheck %s
+;;
+;; "errors" reports we've seen no errors.
+;; CHECK: [fuzz-exec] calling errors
+;; CHECK: [LoggingExternalInterface logging 0]
+
+;; "call0" calls "errors", which logs 0 twice.
+;; CHECK: [fuzz-exec] calling call0
+;; CHECK: [LoggingExternalInterface logging 0]
+;; CHECK: [LoggingExternalInterface logging 0]
+
+;; "call3" calls an invalid index, and logs -1 as an error, and 1 as the total
+;; errors so far.
+;; CHECK: [fuzz-exec] calling call3
+;; CHECK: [LoggingExternalInterface logging -1]
+;; CHECK: [LoggingExternalInterface logging 1]
+
+;; Append another build + run.
+;;
+;; RUN: cp %S/../../../scripts/fuzz_shell.js %t.js
+;; RUN: echo "build(binary);" >> %t.js
+;; RUN: echo "callExports();" >> %t.js
+;; RUN: node %t.js %t.wasm | filecheck %s --check-prefix=APPENDED
+;;
+;; The first part is unchanged from before.
+;; APPENDED: [fuzz-exec] calling errors
+;; APPENDED: [LoggingExternalInterface logging 0]
+;; APPENDED: [fuzz-exec] calling call0
+;; APPENDED: [LoggingExternalInterface logging 0]
+;; APPENDED: [LoggingExternalInterface logging 0]
+;; APPENDED: [fuzz-exec] calling call3
+;; APPENDED: [LoggingExternalInterface logging -1]
+;; APPENDED: [LoggingExternalInterface logging 1]
+
+;; Next, we build the module again, append its exports, and call them all.
+
+;; "errors" from the first module recalls that we errored before.
+;; APPENDED: [fuzz-exec] calling errors
+;; APPENDED: [LoggingExternalInterface logging 1]
+
+;; "call0" calls "errors", and they both log 1.
+;; APPENDED: [fuzz-exec] calling call0
+;; APPENDED: [LoggingExternalInterface logging 1]
+;; APPENDED: [LoggingExternalInterface logging 1]
+
+;; "call3" does *not* error like before, as the later exports provide something
+;; at index 3: the second module's "errors". That reports that the second module
+;; has seen no errors, and then call3 from the first module reports that that
+;; module has seen 1 error.
+;; APPENDED: [fuzz-exec] calling call3
+;; APPENDED: [LoggingExternalInterface logging 0]
+;; APPENDED: [LoggingExternalInterface logging 1]
+
+;; "errors" from the second module reports no errors.
+;; APPENDED: [fuzz-exec] calling errors
+;; APPENDED: [LoggingExternalInterface logging 0]
+
+;; "call0" from the second module to the first makes the first module's "errors"
+;; report 1, and then we report 0 from the second module.
+;; APPENDED: [fuzz-exec] calling call0
+;; APPENDED: [LoggingExternalInterface logging 1]
+;; APPENDED: [LoggingExternalInterface logging 0]
+
+;; "call3" from the second module calls "errors" in the second module, and they
+;; both report 0 errors.
+;; APPENDED: [fuzz-exec] calling call3
+;; APPENDED: [LoggingExternalInterface logging 0]
+;; APPENDED: [LoggingExternalInterface logging 0]
+
+;; Overall, we have seen each module call the other, showing calls work both
+;; ways.
diff --git a/test/unit/test_cluster_fuzz.py b/test/unit/test_cluster_fuzz.py
index 1d275c712..8ec1d8928 100644
--- a/test/unit/test_cluster_fuzz.py
+++ b/test/unit/test_cluster_fuzz.py
@@ -247,6 +247,43 @@ class ClusterFuzz(utils.BinaryenTestCase):
 
         print()
 
+        # To check for interesting JS file contents, we'll note how many times
+        # we build and run the wasm.
+        seen_builds = []
+        seen_calls = []
+
+        for i in range(1, N + 1):
+            fuzz_file = os.path.join(temp_dir.name, f'fuzz-binaryen-{i}.js')
+            with open(fuzz_file) as f:
+                js = f.read()
+            seen_builds.append(js.count('build(binary);'))
+            seen_calls.append(js.count('callExports();'))
+
+        # There is always one build and one call (those are in the default
+        # fuzz_shell.js), and we add a couple of operations, each with equal
+        # probability to be a build or a call, so over the 100 testcases here we
+        # have an overwhelming probability to see at least one extra build and
+        # one extra call.
+        #
+        # builds and calls are distributed as mean 4, stddev 5, median 2.
+        print(f'mean JS builds:   {statistics.mean(seen_builds)}')
+        print(f'stdev JS builds:  {statistics.stdev(seen_builds)}')
+        print(f'median JS builds: {statistics.median(seen_builds)}')
+        # Assert on at least 2, which means we added at least one to the default
+        # one that always exists, as mentioned before.
+        self.assertGreaterEqual(max(seen_builds), 2)
+        self.assertGreater(statistics.stdev(seen_builds), 0)
+
+        print()
+
+        print(f'mean JS calls:   {statistics.mean(seen_calls)}')
+        print(f'stdev JS calls:  {statistics.stdev(seen_calls)}')
+        print(f'median JS calls: {statistics.median(seen_calls)}')
+        self.assertGreaterEqual(max(seen_calls), 2)
+        self.assertGreater(statistics.stdev(seen_calls), 0)
+
+        print()
+
     # "zzz" in test name so that this runs last. If it runs first, it can be
     # confusing as it appears next to the logging of which bundle we use (see
     # setUpClass).