[NFC] Improve ClusterFuzz testing (#7157)

1. Error on retrying due to a wasm-opt issue, locally (in production, we
   don't want to error on ClusterFuzz).

2. Move some asserts from test_run_py to the helper generate_testcases
   (so that the asserts happen in all callers).

2 files changed, 23 insertions, 15 deletions

diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py
index ca7c9e355..2fb64941f 100755
--- a/scripts/fuzz_opt.py
+++ b/scripts/fuzz_opt.py
@@ -1654,10 +1654,15 @@ class ClusterFuzz(TestCaseHandler):
                 os.unlink(f)
 
         # Call run.py(), similarly to how ClusterFuzz does.
-        run([sys.executable,
-             os.path.join(self.clusterfuzz_dir, 'run.py'),
-             '--output_dir=' + os.getcwd(),
-             '--no_of_files=1'])
+        out = run([sys.executable,
+                   os.path.join(self.clusterfuzz_dir, 'run.py'),
+                   '--output_dir=' + os.getcwd(),
+                   '--no_of_files=1'])
+
+        # We should not see any mention of a wasm-opt error that caused a
+        # retry. On production ClusterFuzz this is not an error, but we do want
+        # to know about such issues, as they may be real bugs in wasm-opt.
+        assert 'retry' not in out, out
 
         # We should see the two files.
         assert os.path.exists(fuzz_file)
diff --git a/test/unit/test_cluster_fuzz.py b/test/unit/test_cluster_fuzz.py
index 8f1d18104..5455d6320 100644
--- a/test/unit/test_cluster_fuzz.py
+++ b/test/unit/test_cluster_fuzz.py
@@ -74,22 +74,14 @@ class ClusterFuzz(utils.BinaryenTestCase):
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
         self.assertEqual(proc.returncode, 0)
-        return proc
-
-    # Test the bundled run.py script.
-    def test_run_py(self):
-        temp_dir = tempfile.TemporaryDirectory()
-
-        N = 10
-        proc = self.generate_testcases(N, temp_dir.name)
 
         # We should have logged the creation of N testcases.
         self.assertEqual(proc.stdout.count('Created testcase:'), N)
 
         # We should have actually created them.
         for i in range(0, N + 2):
-            fuzz_file = os.path.join(temp_dir.name, f'fuzz-binaryen-{i}.js')
-            flags_file = os.path.join(temp_dir.name, f'flags-binaryen-{i}.js')
+            fuzz_file = os.path.join(testcase_dir, f'fuzz-binaryen-{i}.js')
+            flags_file = os.path.join(testcase_dir, f'flags-binaryen-{i}.js')
             # We actually emit the range [1, N], so 0 or N+1 should not exist.
             if i >= 1 and i <= N:
                 self.assertTrue(os.path.exists(fuzz_file))
@@ -98,8 +90,19 @@ class ClusterFuzz(utils.BinaryenTestCase):
                 self.assertTrue(not os.path.exists(fuzz_file))
                 self.assertTrue(not os.path.exists(flags_file))
 
+        return proc
+
+    # Test the bundled run.py script.
+    def test_run_py(self):
+        temp_dir = tempfile.TemporaryDirectory()
+
+        N = 10
+        proc = self.generate_testcases(N, temp_dir.name)
+
         # Run.py should report no errors or warnings to stderr, except from
-        # those we know are safe.
+        # those we know are safe (we cannot test this in generate_testcases,
+        # because the caller could do something like set BINARYEN_PASS_DEBUG,
+        # which generates intentional stderr warnings).
         SAFE_WARNINGS = [
             # When we randomly pick no passes to run, this is shown.
             'warning: no passes specified, not doing any work',