Fix pow2 util and avoid pow2 for left shifting in ZeroRemover (#3293)

Fixes a fuzz bug that was triggered by

https://github.com/WebAssembly/binaryen/pull/3015#issuecomment-718001620

but was actually a pre-existing bug in pow2, that that PR just happened
to uncover.

1 files changed, 10 insertions, 8 deletions

diff --git a/src/passes/OptimizeInstructions.cpp b/src/passes/OptimizeInstructions.cpp
index c7219dd3c..d019a983f 100644
--- a/src/passes/OptimizeInstructions.cpp
+++ b/src/passes/OptimizeInstructions.cpp
@@ -1113,8 +1113,8 @@ private:
 
     struct SeekState {
       Expression* curr;
-      int64_t mul;
-      SeekState(Expression* curr, int64_t mul) : curr(curr), mul(mul) {}
+      uint64_t mul;
+      SeekState(Expression* curr, uint64_t mul) : curr(curr), mul(mul) {}
     };
     std::vector<SeekState> seekStack;
     seekStack.emplace_back(binary, 1);
@@ -1124,8 +1124,8 @@ private:
       auto curr = state.curr;
       auto mul = state.mul;
       if (auto* c = curr->dynCast<Const>()) {
-        int64_t value = c->value.getInteger();
-        if (value != 0LL) {
+        uint64_t value = c->value.getInteger();
+        if (value != 0ULL) {
           constant += value * mul;
           constants.push_back(c);
         }
@@ -1147,17 +1147,19 @@ private:
         } else if (binary->op ==
                    Abstract::getBinary(binary->type, Abstract::Shl)) {
           if (auto* c = binary->right->dynCast<Const>()) {
-            seekStack.emplace_back(
-              binary->left, mul * Bits::pow2(Bits::getEffectiveShifts(c)));
+            seekStack.emplace_back(binary->left,
+                                   mul << Bits::getEffectiveShifts(c));
             continue;
           }
         } else if (binary->op ==
                    Abstract::getBinary(binary->type, Abstract::Mul)) {
           if (auto* c = binary->left->dynCast<Const>()) {
-            seekStack.emplace_back(binary->right, mul * c->value.getInteger());
+            seekStack.emplace_back(binary->right,
+                                   mul * (uint64_t)c->value.getInteger());
             continue;
           } else if (auto* c = binary->right->dynCast<Const>()) {
-            seekStack.emplace_back(binary->left, mul * c->value.getInteger());
+            seekStack.emplace_back(binary->left,
+                                   mul * (uint64_t)c->value.getInteger());
             continue;
           }
         }