Fuzzer: Mark runs where over 50% of functions trap as ignored (#6007)

The number of ignored functions is logged out, so this can help us avoid getting
into a situation where many testcases just trap most of the time rather than
doing anything useful.

50% seems a reasonable cutoff. Even if 50% of functions trap, at least we are
getting 50% that don't, so lots of useful work.

1 files changed, 18 insertions, 3 deletions

diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py
index 6362b3c3f..9e7b59329 100755
--- a/scripts/fuzz_opt.py
+++ b/scripts/fuzz_opt.py
@@ -592,9 +592,9 @@ def fix_spec_output(out):
 ignored_vm_runs = 0
 
 
-def note_ignored_vm_run():
+def note_ignored_vm_run(text='(ignore VM run)'):
     global ignored_vm_runs
-    print('(ignore VM run)')
+    print(text)
     ignored_vm_runs += 1
 
 
@@ -718,7 +718,22 @@ class CompareVMs(TestCaseHandler):
             name = 'binaryen interpreter'
 
             def run(self, wasm):
-                return run_bynterp(wasm, ['--fuzz-exec-before'])
+                output = run_bynterp(wasm, ['--fuzz-exec-before'])
+                if output != IGNORE:
+                    calls = output.count(FUZZ_EXEC_CALL_PREFIX)
+                    errors = output.count(TRAP_PREFIX) + output.count(HOST_LIMIT_PREFIX)
+                    if errors > calls / 2:
+                        # A significant amount of execution on this testcase
+                        # simply trapped, and was not very useful, so mark it
+                        # as ignored. Ideally the fuzzer testcases would be
+                        # improved to reduce this number.
+                        #
+                        # Note that we don't change output=IGNORE as there may
+                        # still be useful testing here (up to 50%), so we only
+                        # note that this is a mostly-ignored run, but we do not
+                        # ignore the parts that are useful.
+                        note_ignored_vm_run(f'(testcase mostly ignored: {calls} calls, {errors} errors)')
+                return output
 
             def can_run(self, wasm):
                 return True