[Wasm GC] [TNH] OptimizeInstructions: remove casts leading to comparisons (#4748)

Comparing references does not depend on the cast, so if we are ignoring
traps in traps-never-happen mode then we can remove them.

1 files changed, 38 insertions, 0 deletions

diff --git a/src/passes/OptimizeInstructions.cpp b/src/passes/OptimizeInstructions.cpp
index c4ce8c239..ebe551d0c 100644
--- a/src/passes/OptimizeInstructions.cpp
+++ b/src/passes/OptimizeInstructions.cpp
@@ -1372,6 +1372,10 @@ struct OptimizeInstructions
   }
 
   void visitRefEq(RefEq* curr) {
+    // Equality does not depend on the type, so casts may be removable.
+    skipCast(curr->left, Type::eqref);
+    skipCast(curr->right, Type::eqref);
+
     // Identical references compare equal.
     if (areConsecutiveInputsEqualAndRemovable(curr->left, curr->right)) {
       replaceCurrent(
@@ -1406,6 +1410,36 @@ struct OptimizeInstructions
     }
   }
 
+  // As skipNonNullCast, but skips all casts if we can do so. This is useful in
+  // cases where we don't actually care about the type but just the value, that
+  // is, if casts of the type do not affect our behavior (which is the case in
+  // ref.eq for example).
+  //
+  // |requiredType| is the type we require as the final output here, or a
+  // subtype of it. We will not remove a cast that would leave something that
+  // would break that. If |requiredType| is not provided we will accept any type
+  // there.
+  void skipCast(Expression*& input, Type requiredType = Type::anyref) {
+    // Traps-never-happen mode is a requirement for us to optimize here.
+    if (!getPassOptions().trapsNeverHappen) {
+      return;
+    }
+    while (1) {
+      if (auto* as = input->dynCast<RefAs>()) {
+        if (Type::isSubType(as->value->type, requiredType)) {
+          input = as->value;
+          continue;
+        }
+      } else if (auto* cast = input->dynCast<RefCast>()) {
+        if (!cast->rtt && Type::isSubType(cast->ref->type, requiredType)) {
+          input = cast->ref;
+          continue;
+        }
+      }
+      break;
+    }
+  }
+
   void visitStructGet(StructGet* curr) { skipNonNullCast(curr->ref); }
 
   void visitStructSet(StructSet* curr) {
@@ -1819,6 +1853,10 @@ struct OptimizeInstructions
       return;
     }
 
+    // What the reference points to does not depend on the type, so casts may be
+    // removable.
+    skipCast(curr->value);
+
     // Optimizating RefIs is not that obvious, since even if we know the result
     // evaluates to 0 or 1 then the replacement may not actually save code size,
     // since RefIsNull is a single byte (the others are 2), while adding a Const