[Wasm GC] Optimize casts of null values better (#5423)

Instead of only looking at the final fallthrough value and seeing whether it is
a `RefNull` expression to determine if we are casting a null value, check the
type of each intermediate fallthrough value to see if it is a null reference.

Also improve `evaluateCastCheck` to return `Failure` instead of
`SuccessOnlyIfNonNull` if the cast value is a reference to bottom type, since
those can never be non-null.

1 files changed, 21 insertions, 30 deletions

diff --git a/src/passes/OptimizeInstructions.cpp b/src/passes/OptimizeInstructions.cpp
index bbb497f39..b725ef03f 100644
--- a/src/passes/OptimizeInstructions.cpp
+++ b/src/passes/OptimizeInstructions.cpp
@@ -1914,34 +1914,10 @@ struct OptimizeInstructions
       return;
     }
 
-    Builder builder(*getModule());
-
-    auto fallthrough =
-      Properties::getFallthrough(curr->ref, getPassOptions(), *getModule());
-
-    auto intendedType = curr->type.getHeapType();
-
-    // If the value is a null, then we know a nullable cast will succeed and a
-    // non-nullable cast will fail. Either way, we do not need the cast. We've
-    // already handled the non-nullable case above, so all we have left is a
-    // nullable one.
-    // Note that we have to avoid changing the type when replacing a cast with
-    // its potentially more refined child, e.g.
-    //   (ref.cast null (ref.as_non_null (.. (ref.null)))
-    if (fallthrough->is<RefNull>() && curr->type.isNullable()) {
-      // Replace the expression to drop the input and directly produce the
-      // null.
-      replaceCurrent(builder.makeSequence(builder.makeDrop(curr->ref),
-                                          builder.makeRefNull(intendedType)));
-      return;
-      // TODO: The optimal ordering of this and the other ref.as_non_null
-      //       stuff later down in this functions is unclear and may be worth
-      //       looking into.
-    }
-
-    // Check whether the cast will definitely fail. Look not just at the
-    // fallthrough but all intermediatary fallthrough values as well, as if any
-    // of them has a type that cannot be cast to us, then we will trap, e.g.
+    // Check whether the cast will definitely fail (or succeed). Look not just
+    // at the fallthrough but all intermediatary fallthrough values as well, as
+    // if any of them has a type that cannot be cast to us, then we will trap,
+    // e.g.
     //
     //   (ref.cast $struct-A
     //     (ref.cast $struct-B
@@ -1950,12 +1926,27 @@ struct OptimizeInstructions
     //
     // The fallthrough is the local.get, but the array cast in the middle
     // proves a trap must happen.
+    Builder builder(*getModule());
+    auto nullType = curr->type.getHeapType().getBottom();
     {
       auto* ref = curr->ref;
       while (1) {
         auto result = GCTypeUtils::evaluateCastCheck(ref->type, curr->type);
 
-        if (result == GCTypeUtils::Failure) {
+        if (result == GCTypeUtils::Success) {
+          // The cast will succeed, but we can't just remove the cast and
+          // replace it with `ref` because the intermediate expressions might
+          // have had side effects. We can replace the cast with a drop followed
+          // by a direct return of the value, though.
+          //
+          // TODO: Do this for non-null values as well by storing the value to
+          // return in a tee.
+          if (ref->type.isNull()) {
+            replaceCurrent(builder.makeSequence(builder.makeDrop(curr->ref),
+                                                builder.makeRefNull(nullType)));
+            return;
+          }
+        } else if (result == GCTypeUtils::Failure) {
           // This cast cannot succeed, so it will trap.
           // Make sure to emit a block with the same type as us; leave updating
           // types for other passes.
@@ -1964,7 +1955,7 @@ struct OptimizeInstructions
             curr->type));
           return;
         } else if (result == GCTypeUtils::SuccessOnlyIfNull) {
-          curr->type = Type(intendedType.getBottom(), Nullable);
+          curr->type = Type(nullType, Nullable);
           // Call replaceCurrent() to make us re-optimize this node, as we may
           // have just unlocked further opportunities. (We could just continue
           // down to the rest, but we'd need to do more work to make sure all