Support br_on_cast null (#5397)

As well as br_on_cast_fail null. Unlike the existing br_on_cast* instructions,
these new instructions treat the cast as succeeding when the input is a null.
Update the internal representation of the cast type in `BrOn` expressions to be
a `Type` rather than a `HeapType` so it will include nullability information.

Also update and improve `RemoveUnusedBrs` to handle the new instructions
correctly and optimize in more cases.

5 files changed, 117 insertions, 20 deletions

diff --git a/test/lit/cast-to-basic.wast b/test/lit/cast-to-basic.wast
index c6b8ea9e7..1aa329bf5 100644
--- a/test/lit/cast-to-basic.wast
+++ b/test/lit/cast-to-basic.wast
@@ -56,11 +56,11 @@
     )
   )
 
-  ;; CHECK:      (func $br-fail (type $none_=>_none)
+  ;; CHECK:      (func $br-null (type $none_=>_none)
   ;; CHECK-NEXT:  (drop
   ;; CHECK-NEXT:   (block $label$1 (result dataref)
   ;; CHECK-NEXT:    (drop
-  ;; CHECK-NEXT:     (br_on_cast_fail $label$1 data
+  ;; CHECK-NEXT:     (br_on_cast $label$1 null data
   ;; CHECK-NEXT:      (ref.null none)
   ;; CHECK-NEXT:     )
   ;; CHECK-NEXT:    )
@@ -68,11 +68,36 @@
   ;; CHECK-NEXT:   )
   ;; CHECK-NEXT:  )
   ;; CHECK-NEXT: )
-  (func $br-fail
+  (func $br-null
     (drop
       (block $l (result structref)
         (drop
-          (br_on_cast_fail $l struct
+          (br_on_cast $l null struct
+            (ref.null none)
+          )
+        )
+        (ref.null none)
+      )
+    )
+  )
+
+  ;; CHECK:      (func $br-fail-null (type $none_=>_none)
+  ;; CHECK-NEXT:  (drop
+  ;; CHECK-NEXT:   (block $label$1 (result dataref)
+  ;; CHECK-NEXT:    (drop
+  ;; CHECK-NEXT:     (br_on_cast_fail $label$1 null data
+  ;; CHECK-NEXT:      (ref.null none)
+  ;; CHECK-NEXT:     )
+  ;; CHECK-NEXT:    )
+  ;; CHECK-NEXT:    (ref.null none)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT: )
+  (func $br-fail-null
+    (drop
+      (block $l (result structref)
+        (drop
+          (br_on_cast_fail $l null struct
             (ref.null none)
           )
         )
diff --git a/test/lit/passes/remove-unused-brs-gc.wast b/test/lit/passes/remove-unused-brs-gc.wast
index c448a47c2..5c45b9e7c 100644
--- a/test/lit/passes/remove-unused-brs-gc.wast
+++ b/test/lit/passes/remove-unused-brs-gc.wast
@@ -3,8 +3,13 @@
 ;; RUN:  | filecheck %s
 
 (module
- ;; CHECK:      (type $struct (struct ))
- (type $struct (struct ))
+ (rec
+  ;; CHECK:      (rec
+  ;; CHECK-NEXT:  (type $struct (struct ))
+  (type $struct (struct))
+  ;; CHECK:       (type $struct2 (struct ))
+  (type $struct2 (struct))
+ )
 
  ;; CHECK:      (func $br_on_non_i31-1 (type $none_=>_none)
  ;; CHECK-NEXT:  (drop
@@ -117,7 +122,6 @@
  )
 
  ;; CHECK:      (func $br_on_cast (type $none_=>_ref|$struct|) (result (ref $struct))
- ;; CHECK-NEXT:  (local $temp (ref null $struct))
  ;; CHECK-NEXT:  (block $block (result (ref $struct))
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (br $block
@@ -128,7 +132,6 @@
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $br_on_cast (result (ref $struct))
-  (local $temp (ref null $struct))
   (block $block (result (ref $struct))
    (drop
     ;; This static cast can be computed at compile time: it will definitely be
@@ -141,23 +144,66 @@
   )
  )
 
+ ;; CHECK:      (func $br_on_cast_unrelated (type $none_=>_ref|$struct|) (result (ref $struct))
+ ;; CHECK-NEXT:  (block $block
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (struct.new_default $struct2)
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (unreachable)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ (func $br_on_cast_unrelated (result (ref $struct))
+  (block $block (result (ref $struct))
+   (drop
+    ;; This cast can be computed at compile time: it will definitely fail, so we
+    ;; can remove it.
+    (br_on_cast $block $struct
+     (struct.new $struct2)
+    )
+   )
+   (unreachable)
+  )
+ )
+
  ;; CHECK:      (func $br_on_cast_no (type $none_=>_ref|$struct|) (result (ref $struct))
- ;; CHECK-NEXT:  (local $temp (ref null $struct))
+ ;; CHECK-NEXT:  (local $struct (ref null $struct))
  ;; CHECK-NEXT:  (block $block (result (ref $struct))
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (br_on_cast $block $struct
- ;; CHECK-NEXT:     (ref.null none)
+ ;; CHECK-NEXT:     (local.get $struct)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (unreachable)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $br_on_cast_no (result (ref $struct))
-  (local $temp (ref null $struct))
+  (local $struct (ref null $struct))
   (block $block (result (ref $struct))
    (drop
     (br_on_cast $block $struct
      ;; As above, but now the type is nullable, so we cannot infer anything.
+     (local.get $struct)
+    )
+   )
+   (unreachable)
+  )
+ )
+
+ ;; CHECK:      (func $br_on_cast_nullable (type $none_=>_ref?|$struct|) (result (ref null $struct))
+ ;; CHECK-NEXT:  (block $block (result nullref)
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (br $block
+ ;; CHECK-NEXT:     (ref.null none)
+ ;; CHECK-NEXT:    )
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (unreachable)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ (func $br_on_cast_nullable (result (ref null $struct))
+  (block $block (result (ref null $struct))
+   (drop
+    (br_on_cast $block null $struct
+     ;; As above, but now the cast allows nulls, so we can optimize.
      (ref.null $struct)
     )
    )
@@ -166,7 +212,6 @@
  )
 
  ;; CHECK:      (func $br_on_cast_fail (type $none_=>_ref|$struct|) (result (ref $struct))
- ;; CHECK-NEXT:  (local $temp (ref null $struct))
  ;; CHECK-NEXT:  (block $block
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (struct.new_default $struct)
@@ -175,7 +220,6 @@
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $br_on_cast_fail (result (ref $struct))
-  (local $temp (ref null $struct))
   (block $block (result (ref $struct))
    (drop
     ;; As $br_on_cast, but this checks for a failing cast, so we know it will
@@ -189,6 +233,7 @@
  )
 
  ;; CHECK:      (func $casts-are-costly (type $i32_=>_none) (param $x i32)
+ ;; CHECK-NEXT:  (local $struct (ref null $struct))
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (if (result i32)
  ;; CHECK-NEXT:    (local.get $x)
@@ -213,7 +258,7 @@
  ;; CHECK-NEXT:    (block $something (result anyref)
  ;; CHECK-NEXT:     (drop
  ;; CHECK-NEXT:      (br_on_cast $something $struct
- ;; CHECK-NEXT:       (ref.null none)
+ ;; CHECK-NEXT:       (local.get $struct)
  ;; CHECK-NEXT:      )
  ;; CHECK-NEXT:     )
  ;; CHECK-NEXT:     (ref.null none)
@@ -242,6 +287,7 @@
   ;; We never turn an if into a select if an arm has a cast of any kind, as
   ;; those things involve branches internally, so we'd be adding more than we
   ;; save.
+  (local $struct (ref null $struct))
   (drop
    (if (result i32)
     (local.get $x)
@@ -267,7 +313,7 @@
      (block $something (result anyref)
       (drop
        (br_on_cast $something $struct
-        (ref.null $struct)
+        (local.get $struct)
        )
       )
       (ref.null any)
diff --git a/test/passes/Oz_fuzz-exec_all-features.txt b/test/passes/Oz_fuzz-exec_all-features.txt
index af706cc23..30949def4 100644
--- a/test/passes/Oz_fuzz-exec_all-features.txt
+++ b/test/passes/Oz_fuzz-exec_all-features.txt
@@ -52,7 +52,7 @@
 [LoggingExternalInterface logging 0]
 [LoggingExternalInterface logging 1]
 [LoggingExternalInterface logging 0]
-[LoggingExternalInterface logging 0]
+[LoggingExternalInterface logging 1]
 [fuzz-exec] calling static-br_on_cast
 [LoggingExternalInterface logging 3]
 [fuzz-exec] calling static-br_on_cast_fail
@@ -62,7 +62,7 @@
  (type $void_func (func))
  (type $struct (struct (field (mut i32))))
  (type $i32_=>_none (func (param i32)))
- (type $extendedstruct (struct (field (mut i32)) (field f64)))
+ (type $extendedstruct (struct_subtype (field (mut i32)) (field f64) $struct))
  (type $int_func (func (result i32)))
  (import "fuzzing-support" "log-i32" (func $log (param i32)))
  (export "structs" (func $0))
@@ -329,7 +329,7 @@
    (i32.const 0)
   )
   (call $log
-   (i32.const 0)
+   (i32.const 1)
   )
  )
  (func $21 (type $void_func) (; has Stack IR ;)
@@ -391,7 +391,7 @@
 [LoggingExternalInterface logging 0]
 [LoggingExternalInterface logging 1]
 [LoggingExternalInterface logging 0]
-[LoggingExternalInterface logging 0]
+[LoggingExternalInterface logging 1]
 [fuzz-exec] calling static-br_on_cast
 [LoggingExternalInterface logging 3]
 [fuzz-exec] calling static-br_on_cast_fail
diff --git a/test/passes/Oz_fuzz-exec_all-features.wast b/test/passes/Oz_fuzz-exec_all-features.wast
index bf6767c73..0ce79aa4b 100644
--- a/test/passes/Oz_fuzz-exec_all-features.wast
+++ b/test/passes/Oz_fuzz-exec_all-features.wast
@@ -1,6 +1,6 @@
 (module
  (type $struct (struct (mut i32)))
- (type $extendedstruct (struct (mut i32) f64))
+ (type $extendedstruct (struct_subtype (mut i32) f64 $struct))
  (type $bytes (array (mut i8)))
 
  (type $void_func (func))
diff --git a/test/spec/ref_cast.wast b/test/spec/ref_cast.wast
index 704063ec6..b81671e30 100644
--- a/test/spec/ref_cast.wast
+++ b/test/spec/ref_cast.wast
@@ -103,6 +103,18 @@
     (i32.const 1)
   )
 
+  (func (export "test-br-on-cast-null-struct") (result i32)
+    (drop
+      (block $l (result (ref null struct))
+        (drop
+          (br_on_cast $l null struct (ref.null none))
+        )
+        (return (i32.const 0))
+      )
+    )
+    (i32.const 1)
+  )
+
   (func (export "test-br-on-cast-fail-struct") (result i32)
     (drop
       (block $l (result (ref struct))
@@ -115,6 +127,18 @@
     (i32.const 1)
   )
 
+  (func (export "test-br-on-cast-fail-null-struct") (result i32)
+    (drop
+      (block $l (result (ref struct))
+        (drop
+          (br_on_cast_fail $l null struct (ref.null none))
+        )
+        (return (i32.const 0))
+      )
+    )
+    (i32.const 1)
+  )
+
   (func (export "test-trap-null")
     (drop
       (ref.cast $t0
@@ -131,7 +155,9 @@
 (assert_return (invoke "test-ref-test-any") (i32.const 1))
 (assert_return (invoke "test-ref-cast-struct"))
 (assert_return (invoke "test-br-on-cast-struct") (i32.const 1))
+(assert_return (invoke "test-br-on-cast-null-struct") (i32.const 1))
 (assert_return (invoke "test-br-on-cast-fail-struct") (i32.const 0))
+(assert_return (invoke "test-br-on-cast-fail-null-struct") (i32.const 0))
 (assert_trap (invoke "test-trap-null"))
 
 (assert_invalid