[Wasm GC] Check if ref.eq inputs can possibly be the same (#4780)

For them to be the same we must have a value that can appear on both
sides. If the heap types disallow that, then only null is possible, and if
that is impossible as well then the result must be 0.

1 files changed, 27 insertions, 3 deletions

diff --git a/src/passes/OptimizeInstructions.cpp b/src/passes/OptimizeInstructions.cpp
index 5f18beb73..9d50275b9 100644
--- a/src/passes/OptimizeInstructions.cpp
+++ b/src/passes/OptimizeInstructions.cpp
@@ -1494,11 +1494,35 @@ struct OptimizeInstructions
   }
 
   void visitRefEq(RefEq* curr) {
+    // The types may prove that the same reference cannot appear on both sides.
+    auto leftType = curr->left->type;
+    auto rightType = curr->right->type;
+    if (leftType == Type::unreachable || rightType == Type::unreachable) {
+      // Leave this for DCE.
+      return;
+    }
+    auto leftHeapType = leftType.getHeapType();
+    auto rightHeapType = rightType.getHeapType();
+    auto leftIsHeapSubtype = HeapType::isSubType(leftHeapType, rightHeapType);
+    auto rightIsHeapSubtype = HeapType::isSubType(rightHeapType, leftHeapType);
+    if (!leftIsHeapSubtype && !rightIsHeapSubtype &&
+        (leftType.isNonNullable() || rightType.isNonNullable())) {
+      // The heap types have no intersection, so the only thing that can
+      // possibly appear on both sides is null, but one of the two is non-
+      // nullable, which rules that out. So there is no way that the same
+      // reference can appear on both sides.
+      auto* result =
+        Builder(*getModule()).makeConst(Literal::makeZero(Type::i32));
+      replaceCurrent(getDroppedChildrenAndAppend(
+        curr, *getModule(), getPassOptions(), result));
+      return;
+    }
+
     // Equality does not depend on the type, so casts may be removable.
     //
-    // This is safe to do first because nothing else here cares about the type,
-    // and we consume the two input references, so removing a cast could not
-    // help our parents (see "notes on removing casts").
+    // This is safe to do first because nothing farther down cares about the
+    // type, and we consume the two input references, so removing a cast could
+    // not help our parents (see "notes on removing casts").
     skipCast(curr->left, Type::eqref);
     skipCast(curr->right, Type::eqref);