Reducer: Improve warning on scripts that ignore the input (#3490)

The risk the warning checks for is giving the reducer a script that ignores
the input. To do so it runs the command in the input, and runs it on a
garbage file, and checks if the result is different. However, if the script
does immediately fail on the input - because the input is a crash testcase
or such - then this does not work, as the result on a garbage input may be
the same error.

To avoid that, also check what happens on a trivial valid wasm as input.
Only show the warning if the result on the original input, on a garbage
wasm, and on a trivial wasm, are all the same - in that case, likely the
script really is ignoring the input.

2 files changed, 33 insertions, 9 deletions

diff --git a/src/tools/wasm-reduce.cpp b/src/tools/wasm-reduce.cpp
index 13288c79d..a6bf238fc 100644
--- a/src/tools/wasm-reduce.cpp
+++ b/src/tools/wasm-reduce.cpp
@@ -1199,19 +1199,30 @@ int main(int argc, const char* argv[]) {
                     expected);
   }
 
-  std::cerr
-    << "|checking that command has different behavior on invalid binary (this "
-       "verifies that the test file is used by the command)\n";
-  {
+  if (!force) {
+    std::cerr << "|checking that command has different behavior on different "
+                 "inputs (this "
+                 "verifies that the test file is used by the command)\n";
+    // Try it on an invalid input.
     {
       std::ofstream dst(test, std::ios::binary);
       dst << "waka waka\n";
     }
-    ProgramResult result(command);
-    if (result == expected) {
-      stopIfNotForced(
-        "running command on an invalid module should give different results",
-        result);
+    ProgramResult resultOnInvalid(command);
+    if (resultOnInvalid == expected) {
+      // Try it on a valid input.
+      Module emptyModule;
+      ModuleWriter writer;
+      writer.setBinary(true);
+      writer.write(emptyModule, test);
+      ProgramResult resultOnValid(command);
+      if (resultOnValid == expected) {
+        Fatal()
+          << "running the command on the given input gives the same result as "
+             "when running it on either a trivial valid wasm or a file with "
+             "nonsense in it. does the script not look at the test file (" +
+               test + ")? (use -f to ignore this check)";
+      }
     }
   }
 
diff --git a/test/unit/test_reduce.py b/test/unit/test_reduce.py
new file mode 100644
index 000000000..0abba604f
--- /dev/null
+++ b/test/unit/test_reduce.py
@@ -0,0 +1,13 @@
+import subprocess
+
+from scripts.test import shared
+from . import utils
+
+
+class ReduceTest(utils.BinaryenTestCase):
+    def test_warn_on_no_passes(self):
+        # run a reducer command that does nothing, and so ignores the input
+        open('do_nothing.py', 'w').close()
+        cmd = shared.WASM_REDUCE + [self.input_path('empty.wasm'), '-w', 'w.wasm', '-t', 't.wasm', '--command=python do_nothing.py']
+        err = shared.run_process(cmd, check=False, stderr=subprocess.PIPE).stderr
+        self.assertIn('Fatal: running the command on the given input gives the same result as when running it on either a trivial valid wasm or a file with nonsense in it. does the script not look at the test file (t.wasm)? (use -f to ignore this check)', err)