Fix a bug optimizing out br_on_cast (#5403)

We were considering casts between unrelated types as unconditionally failing,
but in the case where the unrelated types are nullable, the cast could still
succeed if the value is null.

This bug was introduced in #5397.

1 files changed, 95 insertions, 4 deletions

diff --git a/test/lit/passes/remove-unused-brs-gc.wast b/test/lit/passes/remove-unused-brs-gc.wast
index 5c45b9e7c..b4c529f74 100644
--- a/test/lit/passes/remove-unused-brs-gc.wast
+++ b/test/lit/passes/remove-unused-brs-gc.wast
@@ -144,16 +144,29 @@
   )
  )
 
- ;; CHECK:      (func $br_on_cast_unrelated (type $none_=>_ref|$struct|) (result (ref $struct))
- ;; CHECK-NEXT:  (block $block
+ ;; CHECK:      (func $br_on_cast_unrelated (type $none_=>_ref?|$struct|) (result (ref null $struct))
+ ;; CHECK-NEXT:  (local $nullable-struct2 (ref null $struct2))
+ ;; CHECK-NEXT:  (block $block (result (ref null $struct))
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (struct.new_default $struct2)
+ ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (struct.new_default $struct2)
  ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (local.get $nullable-struct2)
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (br_on_cast $block null $struct
+ ;; CHECK-NEXT:     (local.get $nullable-struct2)
+ ;; CHECK-NEXT:    )
+ ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (unreachable)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
- (func $br_on_cast_unrelated (result (ref $struct))
-  (block $block (result (ref $struct))
+ (func $br_on_cast_unrelated (result (ref null $struct))
+  (local $nullable-struct2 (ref null $struct2))
+  (block $block (result (ref null $struct))
    (drop
     ;; This cast can be computed at compile time: it will definitely fail, so we
     ;; can remove it.
@@ -161,6 +174,84 @@
      (struct.new $struct2)
     )
    )
+   (drop
+    ;; We can still remove it even if the cast allows nulls.
+    (br_on_cast $block null $struct
+     (struct.new $struct2)
+    )
+   )
+   (drop
+    ;; Or if the cast does not allow nulls and the value is nullable.
+    (br_on_cast $block $struct
+     (local.get $nullable-struct2)
+    )
+   )
+   (drop
+    ;; But if both are nullable, then we can't optimize because the cast would
+    ;; succeed if the value is a null.
+    (br_on_cast $block null $struct
+     (local.get $nullable-struct2)
+    )
+   )
+   (unreachable)
+  )
+ )
+
+ ;; CHECK:      (func $br_on_cast_fail_unrelated (type $none_=>_anyref) (result anyref)
+ ;; CHECK-NEXT:  (local $nullable-struct2 (ref null $struct2))
+ ;; CHECK-NEXT:  (block $block (result (ref null $struct2))
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (br $block
+ ;; CHECK-NEXT:     (struct.new_default $struct2)
+ ;; CHECK-NEXT:    )
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (br $block
+ ;; CHECK-NEXT:     (struct.new_default $struct2)
+ ;; CHECK-NEXT:    )
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (br $block
+ ;; CHECK-NEXT:     (local.get $nullable-struct2)
+ ;; CHECK-NEXT:    )
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (br_on_cast_fail $block null $struct
+ ;; CHECK-NEXT:     (local.get $nullable-struct2)
+ ;; CHECK-NEXT:    )
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (unreachable)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ (func $br_on_cast_fail_unrelated (result anyref)
+  (local $nullable-struct2 (ref null $struct2))
+  (block $block (result anyref)
+   (drop
+    ;; This cast can be computed at compile time: it will definitely fail, so we
+    ;; can replace it with an unconditional br.
+    (br_on_cast_fail $block $struct
+     (struct.new $struct2)
+    )
+   )
+   (drop
+    ;; We can still replace it even if the cast allows nulls.
+    (br_on_cast_fail $block null $struct
+     (struct.new $struct2)
+    )
+   )
+   (drop
+    ;; Or if the cast does not allow nulls and the value is nullable.
+    (br_on_cast_fail $block $struct
+     (local.get $nullable-struct2)
+    )
+   )
+   (drop
+    ;; But if both are nullable, then we can't optimize because the cast would
+    ;; succeed if the value is a null.
+    (br_on_cast_fail $block null $struct
+     (local.get $nullable-struct2)
+    )
+   )
    (unreachable)
   )
  )