Fix fuzz bug in RemoveUnusedBrs with incremental type updating (#4309)

The BrOn logic there is incremental in optimizing and updating types, and so
we cannot assume that at every point in the middle the types are fully
updated.

2 files changed, 37 insertions, 2 deletions

diff --git a/src/passes/RemoveUnusedBrs.cpp b/src/passes/RemoveUnusedBrs.cpp
index 01c465f18..f383f669a 100644
--- a/src/passes/RemoveUnusedBrs.cpp
+++ b/src/passes/RemoveUnusedBrs.cpp
@@ -694,8 +694,13 @@ struct RemoveUnusedBrs : public WalkerPass<PostWalker<RemoveUnusedBrs>> {
       bool worked = false;
 
       void visitBrOn(BrOn* curr) {
-        // Ignore unreachable BrOns which we cannot improve anyhow.
-        if (curr->type == Type::unreachable) {
+        // Ignore unreachable BrOns which we cannot improve anyhow. Note that
+        // we must check the ref field manually, as we may be changing types as
+        // we go here. (Another option would be to use a TypeUpdater here
+        // instead of calling ReFinalize at the very end, but that would be more
+        // complex and slower.)
+        if (curr->type == Type::unreachable ||
+            curr->ref->type == Type::unreachable) {
           return;
         }
 
diff --git a/test/lit/passes/remove-unused-brs-gc.wast b/test/lit/passes/remove-unused-brs-gc.wast
index e38eccebd..10b316900 100644
--- a/test/lit/passes/remove-unused-brs-gc.wast
+++ b/test/lit/passes/remove-unused-brs-gc.wast
@@ -3,6 +3,9 @@
 ;; RUN:  | filecheck %s
 
 (module
+ ;; CHECK:      (type $struct (struct ))
+ (type $struct (struct ))
+
  ;; CHECK:      (func $br_on_non_data-1
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block $any (result anyref)
@@ -82,4 +85,31 @@
    )
   )
  )
+
+ ;; CHECK:      (func $nested_br_on (result dataref)
+ ;; CHECK-NEXT:  (block $label$1 (result (ref $struct))
+ ;; CHECK-NEXT:   (drop
+ ;; CHECK-NEXT:    (br $label$1
+ ;; CHECK-NEXT:     (struct.new_default $struct)
+ ;; CHECK-NEXT:    )
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:   (unreachable)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ (func $nested_br_on (result dataref)
+  (block $label$1 (result dataref)
+   (drop
+    ;; The inner br_on_data will become a direct br since the type proves it
+    ;; is in fact data. That then becomes unreachable, and the parent must
+    ;; handle that properly (do nothing without hitting an assertion).
+    (br_on_data $label$1
+     (br_on_data $label$1
+      (struct.new_default $struct)
+     )
+    )
+   )
+   (unreachable)
+  )
+ )
 )
+