[Wasm GC] Optimize casts of null values better (#5423)

Instead of only looking at the final fallthrough value and seeing whether it is
a `RefNull` expression to determine if we are casting a null value, check the
type of each intermediate fallthrough value to see if it is a null reference.

Also improve `evaluateCastCheck` to return `Failure` instead of
`SuccessOnlyIfNonNull` if the cast value is a reference to bottom type, since
those can never be non-null.

1 files changed, 54 insertions, 28 deletions

diff --git a/test/lit/passes/optimize-instructions-gc.wast b/test/lit/passes/optimize-instructions-gc.wast
index 8aa9e36f7..c4d78d12d 100644
--- a/test/lit/passes/optimize-instructions-gc.wast
+++ b/test/lit/passes/optimize-instructions-gc.wast
@@ -1887,24 +1887,30 @@
   ;; CHECK-NEXT:  (drop
   ;; CHECK-NEXT:   (block (result nullref)
   ;; CHECK-NEXT:    (drop
-  ;; CHECK-NEXT:     (ref.null none)
+  ;; CHECK-NEXT:     (local.tee $a
+  ;; CHECK-NEXT:      (ref.null none)
+  ;; CHECK-NEXT:     )
   ;; CHECK-NEXT:    )
   ;; CHECK-NEXT:    (ref.null none)
   ;; CHECK-NEXT:   )
   ;; CHECK-NEXT:  )
   ;; CHECK-NEXT:  (drop
-  ;; CHECK-NEXT:   (block (result nullref)
+  ;; CHECK-NEXT:   (block
   ;; CHECK-NEXT:    (drop
-  ;; CHECK-NEXT:     (ref.null none)
+  ;; CHECK-NEXT:     (local.tee $a
+  ;; CHECK-NEXT:      (ref.null none)
+  ;; CHECK-NEXT:     )
   ;; CHECK-NEXT:    )
-  ;; CHECK-NEXT:    (ref.null none)
+  ;; CHECK-NEXT:    (unreachable)
   ;; CHECK-NEXT:   )
   ;; CHECK-NEXT:  )
   ;; CHECK-NEXT:  (drop
   ;; CHECK-NEXT:   (block (result nullref)
   ;; CHECK-NEXT:    (drop
-  ;; CHECK-NEXT:     (local.tee $a
-  ;; CHECK-NEXT:      (ref.null none)
+  ;; CHECK-NEXT:     (block (result nullref)
+  ;; CHECK-NEXT:      (ref.cast null none
+  ;; CHECK-NEXT:       (local.get $a)
+  ;; CHECK-NEXT:      )
   ;; CHECK-NEXT:     )
   ;; CHECK-NEXT:    )
   ;; CHECK-NEXT:    (ref.null none)
@@ -1913,8 +1919,10 @@
   ;; CHECK-NEXT:  (drop
   ;; CHECK-NEXT:   (block
   ;; CHECK-NEXT:    (drop
-  ;; CHECK-NEXT:     (local.tee $a
-  ;; CHECK-NEXT:      (ref.null none)
+  ;; CHECK-NEXT:     (block (result nullref)
+  ;; CHECK-NEXT:      (ref.cast null none
+  ;; CHECK-NEXT:       (local.get $a)
+  ;; CHECK-NEXT:      )
   ;; CHECK-NEXT:     )
   ;; CHECK-NEXT:    )
   ;; CHECK-NEXT:    (unreachable)
@@ -1934,24 +1942,30 @@
   ;; NOMNL-NEXT:  (drop
   ;; NOMNL-NEXT:   (block (result nullref)
   ;; NOMNL-NEXT:    (drop
-  ;; NOMNL-NEXT:     (ref.null none)
+  ;; NOMNL-NEXT:     (local.tee $a
+  ;; NOMNL-NEXT:      (ref.null none)
+  ;; NOMNL-NEXT:     )
   ;; NOMNL-NEXT:    )
   ;; NOMNL-NEXT:    (ref.null none)
   ;; NOMNL-NEXT:   )
   ;; NOMNL-NEXT:  )
   ;; NOMNL-NEXT:  (drop
-  ;; NOMNL-NEXT:   (block (result nullref)
+  ;; NOMNL-NEXT:   (block
   ;; NOMNL-NEXT:    (drop
-  ;; NOMNL-NEXT:     (ref.null none)
+  ;; NOMNL-NEXT:     (local.tee $a
+  ;; NOMNL-NEXT:      (ref.null none)
+  ;; NOMNL-NEXT:     )
   ;; NOMNL-NEXT:    )
-  ;; NOMNL-NEXT:    (ref.null none)
+  ;; NOMNL-NEXT:    (unreachable)
   ;; NOMNL-NEXT:   )
   ;; NOMNL-NEXT:  )
   ;; NOMNL-NEXT:  (drop
   ;; NOMNL-NEXT:   (block (result nullref)
   ;; NOMNL-NEXT:    (drop
-  ;; NOMNL-NEXT:     (local.tee $a
-  ;; NOMNL-NEXT:      (ref.null none)
+  ;; NOMNL-NEXT:     (block (result nullref)
+  ;; NOMNL-NEXT:      (ref.cast null none
+  ;; NOMNL-NEXT:       (local.get $a)
+  ;; NOMNL-NEXT:      )
   ;; NOMNL-NEXT:     )
   ;; NOMNL-NEXT:    )
   ;; NOMNL-NEXT:    (ref.null none)
@@ -1960,8 +1974,10 @@
   ;; NOMNL-NEXT:  (drop
   ;; NOMNL-NEXT:   (block
   ;; NOMNL-NEXT:    (drop
-  ;; NOMNL-NEXT:     (local.tee $a
-  ;; NOMNL-NEXT:      (ref.null none)
+  ;; NOMNL-NEXT:     (block (result nullref)
+  ;; NOMNL-NEXT:      (ref.cast null none
+  ;; NOMNL-NEXT:       (local.get $a)
+  ;; NOMNL-NEXT:      )
   ;; NOMNL-NEXT:     )
   ;; NOMNL-NEXT:    )
   ;; NOMNL-NEXT:    (unreachable)
@@ -1973,32 +1989,42 @@
     ;; Casting nulls results in a null.
     (drop
       (ref.cast null $A
-        (ref.null $A)
+        (ref.null none)
       )
     )
+    ;; A fallthrough works too.
     (drop
       (ref.cast null $A
-        (ref.null $B)
+        (local.tee $a
+          (ref.null none)
+        )
       )
     )
+    ;; A non-null cast of a falling-though null will trap.
     (drop
-      (ref.cast null $B
-        (ref.null $A)
+      (ref.cast $A
+        (local.tee $a
+          (ref.null none)
+        )
       )
     )
-    ;; A fallthrough works too.
+    ;; The prior two examples work even if the fallthrough is only later proven
+    ;; to be null.
     (drop
-      (ref.cast null $A
-        (local.tee $a
-          (ref.null $A)
+      (ref.cast null $B
+        (block (result (ref null $A))
+          (ref.cast null none
+            (local.get $a)
+          )
         )
       )
     )
-    ;; A non-null cast of a falling-though null will trap.
     (drop
-      (ref.cast $A
-        (local.tee $a
-          (ref.null $A)
+      (ref.cast $B
+        (block (result (ref null $A))
+          (ref.cast null none
+            (local.get $a)
+          )
         )
       )
     )