[Wasm GC] Optimize RefCast + ignore-implicit-traps (#3748)

If we are ignoring implicit traps, and if the cast is from a subtype to a supertype,
then we ignore the possible RTT-related inconsistency and can just drop the
cast.

See #3636

2 files changed, 175 insertions, 0 deletions

diff --git a/src/passes/OptimizeInstructions.cpp b/src/passes/OptimizeInstructions.cpp
index 972ffd794..e71506e28 100644
--- a/src/passes/OptimizeInstructions.cpp
+++ b/src/passes/OptimizeInstructions.cpp
@@ -1006,6 +1006,42 @@ struct OptimizeInstructions
     }
   }
 
+  void visitRefCast(RefCast* curr) {
+    if (curr->type == Type::unreachable) {
+      return;
+    }
+    if (getPassOptions().ignoreImplicitTraps) {
+      // A ref.cast traps when the RTTs do not line up, which can be of one of
+      // two sorts of issues:
+      //  1. The value being cast is not even a subtype of the cast type. In
+      //     that case the RTTs trivially cannot indicate subtyping, because
+      //     RTT subtyping is a subset of static subtyping. For example, maybe
+      //     we are trying to cast a {i32} struct to an [f64] array.
+      //  2. The value is a subtype of the cast type, but the RTTs still do not
+      //     fit. That indicates a difference between RTT subtyping and static
+      //     subtyping. That is, the type may be right but the chain of rtt.subs
+      //     is not.
+      // If we ignore implicit traps then we would like to assume that neither
+      // of those two situations can happen. However, we still cannot do
+      // anything if point 1 is a problem, that is, if the value is not a
+      // subtype of the cast type, as we can't remove the cast in that case -
+      // the wasm would not validate. But if the type *is* a subtype, then we
+      // can ignore a possible trap on 2 and remove it.
+      //
+      // We also do not do this if the arguments cannot be reordered. If we
+      // can't do that then we need to add a drop, at minimum (which may still
+      // be worthwhile, but depends on other optimizations kicking in, so it's
+      // not clearly worthwhile).
+      if (HeapType::isSubType(curr->ref->type.getHeapType(),
+                              curr->rtt->type.getHeapType()) &&
+          canReorder(curr->ref, curr->rtt)) {
+        Builder builder(*getModule());
+        replaceCurrent(
+          builder.makeSequence(builder.makeDrop(curr->rtt), curr->ref));
+      }
+    }
+  }
+
   void visitRefIs(RefIs* curr) {
     if (curr->type == Type::unreachable) {
       return;
diff --git a/test/lit/passes/optimize-instructions-gc-iit.wast b/test/lit/passes/optimize-instructions-gc-iit.wast
new file mode 100644
index 000000000..10bbbfc4a
--- /dev/null
+++ b/test/lit/passes/optimize-instructions-gc-iit.wast
@@ -0,0 +1,139 @@
+;; NOTE: Assertions have been generated by update_lit_checks.py and should not be edited.
+;; RUN: wasm-opt %s --optimize-instructions --ignore-implicit-traps --enable-reference-types --enable-gc -S -o - \
+;; RUN:   | filecheck %s
+
+(module
+  (type $parent (struct (field i32)))
+  (type $child  (struct (field i32) (field f64)))
+  (type $other  (struct (field i64) (field f32)))
+
+  (func $foo)
+
+  ;; CHECK:      (func $ref-cast-iit (param $parent (ref $parent)) (param $child (ref $child)) (param $other (ref $other)) (param $parent-rtt (rtt $parent)) (param $child-rtt (rtt $child)) (param $other-rtt (rtt $other))
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (block (result (ref $parent))
+  ;; CHECK-NEXT:    (drop
+  ;; CHECK-NEXT:     (local.get $parent-rtt)
+  ;; CHECK-NEXT:    )
+  ;; CHECK-NEXT:    (local.get $parent)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (block (result (ref $child))
+  ;; CHECK-NEXT:    (drop
+  ;; CHECK-NEXT:     (local.get $parent-rtt)
+  ;; CHECK-NEXT:    )
+  ;; CHECK-NEXT:    (local.get $child)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (ref.cast
+  ;; CHECK-NEXT:    (local.get $parent)
+  ;; CHECK-NEXT:    (local.get $child-rtt)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (ref.cast
+  ;; CHECK-NEXT:    (local.get $child)
+  ;; CHECK-NEXT:    (local.get $other-rtt)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT: )
+  (func $ref-cast-iit
+    (param $parent (ref $parent))
+    (param $child (ref $child))
+    (param $other (ref $other))
+
+    (param $parent-rtt (rtt $parent))
+    (param $child-rtt (rtt $child))
+    (param $other-rtt (rtt $other))
+
+    ;; a cast of parent to an rtt of parent: static subtyping matches.
+    (drop
+      (ref.cast
+        (local.get $parent)
+        (local.get $parent-rtt)
+      )
+    )
+    ;; a cast of child to a supertype: static subtyping matches.
+    (drop
+      (ref.cast
+        (local.get $child)
+        (local.get $parent-rtt)
+      )
+    )
+    ;; a cast of parent to a subtype: static subtyping does not match.
+    (drop
+      (ref.cast
+        (local.get $parent)
+        (local.get $child-rtt)
+      )
+    )
+    ;; a cast of child to an unrelated type: static subtyping does not match.
+    (drop
+      (ref.cast
+        (local.get $child)
+        (local.get $other-rtt)
+      )
+    )
+  )
+
+  ;; CHECK:      (func $ref-cast-iit-bad (param $parent (ref $parent)) (param $parent-rtt (rtt $parent))
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (ref.cast
+  ;; CHECK-NEXT:    (block $block (result (ref $parent))
+  ;; CHECK-NEXT:     (call $foo)
+  ;; CHECK-NEXT:     (local.get $parent)
+  ;; CHECK-NEXT:    )
+  ;; CHECK-NEXT:    (block $block0 (result (rtt $parent))
+  ;; CHECK-NEXT:     (call $foo)
+  ;; CHECK-NEXT:     (local.get $parent-rtt)
+  ;; CHECK-NEXT:    )
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (ref.cast
+  ;; CHECK-NEXT:    (local.get $parent)
+  ;; CHECK-NEXT:    (unreachable)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (ref.cast
+  ;; CHECK-NEXT:    (unreachable)
+  ;; CHECK-NEXT:    (local.get $parent-rtt)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT: )
+  (func $ref-cast-iit-bad
+    (param $parent (ref $parent))
+    (param $parent-rtt (rtt $parent))
+
+    ;; ignore due to the inability to reorder
+    (drop
+      (ref.cast
+        (block (result (ref $parent))
+          (call $foo)
+          (local.get $parent)
+        )
+        (block (result (rtt $parent))
+          (call $foo)
+          (local.get $parent-rtt)
+        )
+      )
+    )
+
+    ;; ignore unreachability
+    (drop
+      (ref.cast
+        (local.get $parent)
+        (unreachable)
+      )
+    )
+    (drop
+      (ref.cast
+        (unreachable)
+        (local.get $parent-rtt)
+      )
+    )
+  )
+)