wasm2js: Avoid emitting non-JS code during opt (#5378)

As noted in #4806, trying to optimize past level 0 can result in
passes emitting non-JS code, which is then unable to be converted during
final output.

This commit creates a new targetJS option in PassOptions, which can
be checked inside each pass where non-JS code might be emitted.

This commit initially adds that logic to OptimizeInstructions, where
this issue was first noticed.

3 files changed, 13 insertions, 1 deletions

diff --git a/src/pass.h b/src/pass.h
index 4ab0a8c34..36d49f7b7 100644
--- a/src/pass.h
+++ b/src/pass.h
@@ -217,6 +217,12 @@ struct PassOptions {
   bool closedWorld = false;
   // Whether to try to preserve debug info through, which are special calls.
   bool debugInfo = false;
+  // Whether we are targeting JS. In that case we want to avoid emitting things
+  // in the optimizer that do not translate well to JS, or that could cause us
+  // to need extra lowering work or even a loop (where we optimize to something
+  // that needs lowering, then we lower it, then we can optimize it again to the
+  // original form).
+  bool targetJS = false;
   // Arbitrary string arguments from the commandline, which we forward to
   // passes.
   std::unordered_map<std::string, std::string> arguments;
diff --git a/src/passes/OptimizeInstructions.cpp b/src/passes/OptimizeInstructions.cpp
index e61373711..ad51d7cd4 100644
--- a/src/passes/OptimizeInstructions.cpp
+++ b/src/passes/OptimizeInstructions.cpp
@@ -3669,7 +3669,10 @@ private:
     {
       // ~(1 << x) aka (1 << x) ^ -1  ==>  rotl(-2, x)
       Expression* x;
-      if (matches(curr, binary(Xor, binary(Shl, ival(1), any(&x)), ival(-1)))) {
+      // Note that we avoid this in JS mode, as emitting a rotation would
+      // require lowering that rotation for JS in another cycle of work.
+      if (matches(curr, binary(Xor, binary(Shl, ival(1), any(&x)), ival(-1))) &&
+          !getPassOptions().targetJS) {
         curr->op = Abstract::getBinary(type, RotL);
         right->value = Literal::makeFromInt32(-2, type);
         curr->left = right;
diff --git a/src/wasm2js.h b/src/wasm2js.h
index 982184510..db394c36c 100644
--- a/src/wasm2js.h
+++ b/src/wasm2js.h
@@ -173,8 +173,11 @@ public:
     // and store would need to do a check. Given that, we can just ignore
     // implicit traps like those when optimizing. (When not optimizing, it's
     // nice to see codegen that matches wasm more precisely.)
+    // It is also important to prevent the optimizer from adding new things that
+    // require additional lowering, as we could hit a cycle.
     if (options.optimizeLevel > 0) {
       options.ignoreImplicitTraps = true;
+      options.targetJS = true;
     }
   }