Improve cast optimizations (#5876)

Simplify the optimization of ref.cast and ref.test in OptimizeInstructions by
moving the loop that examines fallthrough values one at a time out to a shared
function in properties.h. Also simplify ref.cast optimization by analyzing the
cast result in just one place.

In addition to simplifying the code, also make the cast optimizations more
powerful by analyzing the nullability and heap type of the cast value
independently, resulting in a potentially more precise analysis of the cast
behavior. Also improve optimization power by considering fallthrough values when
optimizing the SuccessOnlyIfNonNull case.

1 files changed, 68 insertions, 0 deletions

diff --git a/src/ir/properties.h b/src/ir/properties.h
index d47cf774b..7f247f72f 100644
--- a/src/ir/properties.h
+++ b/src/ir/properties.h
@@ -362,6 +362,74 @@ inline Expression* getFallthrough(
   }
 }
 
+// Look at all the intermediate fallthrough expressions and return the most
+// precise type we know this value will have.
+inline Type getFallthroughType(Expression* curr,
+                               const PassOptions& passOptions,
+                               Module& module) {
+  Type type = curr->type;
+  if (!type.isRef()) {
+    // Only reference types can be improved (excepting improvements to
+    // unreachable, which we leave to refinalization).
+    // TODO: Handle tuples if that ever becomes important.
+    return type;
+  }
+  while (1) {
+    auto* next = getImmediateFallthrough(curr, passOptions, module);
+    if (next == curr) {
+      return type;
+    }
+    type = Type::getGreatestLowerBound(type, next->type);
+    if (type == Type::unreachable) {
+      return type;
+    }
+    curr = next;
+  }
+}
+
+// Find the best fallthrough value ordered by refinement of heaptype, refinement
+// of nullability, and closeness to the current expression. The type of the
+// expression this function returns may be nullable even if `getFallthroughType`
+// is non-nullable, but the heap type will definitely match.
+inline Expression** getMostRefinedFallthrough(Expression** currp,
+                                              const PassOptions& passOptions,
+                                              Module& module) {
+  Expression* curr = *currp;
+  if (!curr->type.isRef()) {
+    return currp;
+  }
+  auto bestType = curr->type.getHeapType();
+  auto bestNullability = curr->type.getNullability();
+  auto** bestp = currp;
+  while (1) {
+    curr = *currp;
+    auto** nextp =
+      Properties::getImmediateFallthroughPtr(currp, passOptions, module);
+    auto* next = *nextp;
+    if (next == curr || next->type == Type::unreachable) {
+      return bestp;
+    }
+    assert(next->type.isRef());
+    auto nextType = next->type.getHeapType();
+    auto nextNullability = next->type.getNullability();
+    if (nextType == bestType) {
+      // Heap types match: refine nullability if possible.
+      if (bestNullability == Nullable && nextNullability == NonNullable) {
+        bestp = nextp;
+        bestNullability = NonNullable;
+      }
+    } else {
+      // Refine heap type if possible, resetting nullability.
+      if (HeapType::isSubType(nextType, bestType)) {
+        bestp = nextp;
+        bestNullability = nextNullability;
+        bestType = nextType;
+      }
+    }
+    currp = nextp;
+  }
+}
+
 inline Index getNumChildren(Expression* curr) {
   Index ret = 0;