[OptimizeInstructions] Simplify floating point ops with NaN on right side (#4985)

x + nan     ->    nan'
x - nan     ->    nan'
x * nan     ->    nan'
x / nan     ->    nan'
min(x, nan) ->    nan'
max(x, nan) ->    nan'

where nan' is canonicalized nan of rhs

x != nan    ->    1
x == nan    ->    0
x >= nan    ->    0
x <= nan    ->    0
x >  nan    ->    0
x <  nan    ->    0

4 files changed, 71 insertions, 28 deletions

diff --git a/src/ir/abstract.h b/src/ir/abstract.h
index 0cfeeea77..04b04e342 100644
--- a/src/ir/abstract.h
+++ b/src/ir/abstract.h
@@ -34,7 +34,6 @@ enum Op {
   Mul,
   DivU,
   DivS,
-  Rem,
   RemU,
   RemS,
   Shl,
@@ -45,6 +44,7 @@ enum Op {
   And,
   Or,
   Xor,
+  CopySign,
   // Relational
   EqZ,
   Eq,
@@ -261,6 +261,8 @@ inline BinaryOp getBinary(Type type, Op op) {
           return DivFloat32;
         case DivS:
           return DivFloat32;
+        case CopySign:
+          return CopySignFloat32;
         case Eq:
           return EqFloat32;
         case Ne:
@@ -282,6 +284,8 @@ inline BinaryOp getBinary(Type type, Op op) {
           return DivFloat64;
         case DivS:
           return DivFloat64;
+        case CopySign:
+          return CopySignFloat64;
         case Eq:
           return EqFloat64;
         case Ne:
diff --git a/src/literal.h b/src/literal.h
index ffd565c71..9d9630ec4 100644
--- a/src/literal.h
+++ b/src/literal.h
@@ -706,6 +706,29 @@ struct GCData {
   GCData(HeapType type, Literals values) : type(type), values(values) {}
 };
 
+// Wasm has nondeterministic rules for NaN propagation in some operations. For
+// example. f32.neg is deterministic and just flips the sign, even of a NaN, but
+// f32.add is nondeterministic, and if one or more of the inputs is a NaN, then
+//
+//  * if all NaNs are canonical NaNs, the output is some arbitrary canonical NaN
+//  * otherwise the output is some arbitrary arithmetic NaN
+//
+// (canonical = NaN payload is 1000..000; arithmetic: 1???..???, that is, the
+// high bit is 1 and all others can be 0 or 1)
+//
+// For many things we don't need to care, and can just do a normal C++ add for
+// an f32.add, for example - the wasm rules are specified so that things like
+// that just work (in order for such math to be fast). However, for our
+// optimizer, it is useful to "standardize" NaNs when there is nondeterminism.
+// That is, when there are multiple valid outputs, it's nice to emit the same
+// one consistently, so that it doesn't look like the optimization changed
+// something. In other words, if the valid output of an expression is a set of
+// valid NaNs, and after optimization the output is still that same set, then
+// the optimization is valid. And if the interpreter picks the same NaN in both
+// cases from that identical set then nothing looks wrong to the fuzzer.
+Literal standardizeNaN(float result);
+Literal standardizeNaN(double result);
+
 } // namespace wasm
 
 namespace std {
diff --git a/src/passes/OptimizeInstructions.cpp b/src/passes/OptimizeInstructions.cpp
index 7f0b28574..9024d0cb6 100644
--- a/src/passes/OptimizeInstructions.cpp
+++ b/src/passes/OptimizeInstructions.cpp
@@ -3597,6 +3597,40 @@ private:
         }
       }
     }
+    {
+      //   x !=  NaN   ==>   1
+      //   x <=> NaN   ==>   0
+      //   x op  NaN'  ==>   NaN',  iff `op` != `copysign` and `x` != C
+      Const* c;
+      Binary* bin;
+      Expression* x;
+      if (matches(curr, binary(&bin, pure(&x), fval(&c))) &&
+          std::isnan(c->value.getFloat()) &&
+          bin->op != getBinary(x->type, CopySign)) {
+        if (bin->isRelational()) {
+          // reuse "c" (nan) constant
+          c->type = Type::i32;
+          if (bin->op == getBinary(x->type, Ne)) {
+            // x != NaN  ==>  1
+            c->value = Literal::makeOne(Type::i32);
+          } else {
+            // x == NaN,
+            // x >  NaN,
+            // x <= NaN
+            // x .. NaN  ==>  0
+            c->value = Literal::makeZero(Type::i32);
+          }
+          return c;
+        }
+        // propagate NaN of RHS but canonicalize it
+        if (c->type == Type::f32) {
+          c->value = standardizeNaN(c->value.getf32());
+        } else {
+          c->value = standardizeNaN(c->value.getf64());
+        }
+        return c;
+      }
+    }
     return nullptr;
   }
 
diff --git a/src/wasm/literal.cpp b/src/wasm/literal.cpp
index 684108581..843f4607f 100644
--- a/src/wasm/literal.cpp
+++ b/src/wasm/literal.cpp
@@ -859,38 +859,20 @@ Literal Literal::demote() const {
   return Literal(float(getf64()));
 }
 
-// Wasm has nondeterministic rules for NaN propagation in some operations. For
-// example. f32.neg is deterministic and just flips the sign, even of a NaN, but
-// f32.add is nondeterministic, and if one or more of the inputs is a NaN, then
-//
-//  * if all NaNs are canonical NaNs, the output is some arbitrary canonical NaN
-//  * otherwise the output is some arbitrary arithmetic NaN
-//
-// (canonical = NaN payload is 1000..000; arithmetic: 1???..???, that is, the
-// high bit is 1 and all others can be 0 or 1)
-//
-// For many things we don't need to care, and can just do a normal C++ add for
-// an f32.add, for example - the wasm rules are specified so that things like
-// that just work (in order for such math to be fast). However, for our
-// optimizer, it is useful to "standardize" NaNs when there is nondeterminism.
-// That is, when there are multiple valid outputs, it's nice to emit the same
-// one consistently, so that it doesn't look like the optimization changed
-// something. In other words, if the valid output of an expression is a set of
-// valid NaNs, and after optimization the output is still that same set, then
-// the optimization is valid. And if the interpreter picks the same NaN in both
-// cases from that identical set then nothing looks wrong to the fuzzer.
-template<typename T> static Literal standardizeNaN(T result) {
+Literal standardizeNaN(float result) {
   if (!std::isnan(result)) {
     return Literal(result);
   }
   // Pick a simple canonical payload, and positive.
-  if (sizeof(T) == 4) {
-    return Literal(Literal(uint32_t(0x7fc00000u)).reinterpretf32());
-  } else if (sizeof(T) == 8) {
-    return Literal(Literal(uint64_t(0x7ff8000000000000ull)).reinterpretf64());
-  } else {
-    WASM_UNREACHABLE("invalid float");
+  return Literal(Literal(uint32_t(0x7fc00000u)).reinterpretf32());
+}
+
+Literal standardizeNaN(double result) {
+  if (!std::isnan(result)) {
+    return Literal(result);
   }
+  // Pick a simple canonical payload, and positive.
+  return Literal(Literal(uint64_t(0x7ff8000000000000ull)).reinterpretf64());
 }
 
 Literal Literal::add(const Literal& other) const {