[NFC] Improve ClusterFuzz testing (#7157)

1. Error on retrying due to a wasm-opt issue, locally (in production, we
   don't want to error on ClusterFuzz).

2. Move some asserts from test_run_py to the helper generate_testcases
   (so that the asserts happen in all callers).

1 files changed, 9 insertions, 4 deletions

diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py
index ca7c9e355..2fb64941f 100755
--- a/scripts/fuzz_opt.py
+++ b/scripts/fuzz_opt.py
@@ -1654,10 +1654,15 @@ class ClusterFuzz(TestCaseHandler):
                 os.unlink(f)
 
         # Call run.py(), similarly to how ClusterFuzz does.
-        run([sys.executable,
-             os.path.join(self.clusterfuzz_dir, 'run.py'),
-             '--output_dir=' + os.getcwd(),
-             '--no_of_files=1'])
+        out = run([sys.executable,
+                   os.path.join(self.clusterfuzz_dir, 'run.py'),
+                   '--output_dir=' + os.getcwd(),
+                   '--no_of_files=1'])
+
+        # We should not see any mention of a wasm-opt error that caused a
+        # retry. On production ClusterFuzz this is not an error, but we do want
+        # to know about such issues, as they may be real bugs in wasm-opt.
+        assert 'retry' not in out, out
 
         # We should see the two files.
         assert os.path.exists(fuzz_file)