Propagate public visibility through all types (#7105)

Previously the classification of public types propagated public
visibility only through types that had previously been collected by
`collectHeapTypes`. Since there are settings that cause
`collectHeapTypes` to collect fewer types, it was possible for public
types to be missed if they were only public because they were reached by
an uncollected types.

Ensure that all public heap types are properly classified by propagating
public visibility even through types that are not part of the collected
output.

Fixes #7103.

2 files changed, 63 insertions, 11 deletions

diff --git a/src/ir/module-utils.cpp b/src/ir/module-utils.cpp
index 2fd129a9c..5ebd6edef 100644
--- a/src/ir/module-utils.cpp
+++ b/src/ir/module-utils.cpp
@@ -598,26 +598,23 @@ void classifyTypeVisibility(Module& wasm,
   // We will need to traverse the types used by public types and mark them
   // public as well.
   std::vector<HeapType> workList;
+  std::unordered_set<RecGroup> publicGroups;
 
   auto notePublic = [&](HeapType type) {
     if (type.isBasic()) {
-      return false;
+      return;
+    }
+    auto group = type.getRecGroup();
+    if (!publicGroups.insert(group).second) {
+      // The groups in this type have already been marked public.
+      return;
     }
-    // All the rec group members are public as well.
-    bool inserted = false;
     for (auto member : type.getRecGroup()) {
       if (auto it = types.find(member); it != types.end()) {
-        if (it->second.visibility == Visibility::Public) {
-          // Since we mark all elements of a group public at once, if there is a
-          // member that is already public, all members must already be public.
-          break;
-        }
         it->second.visibility = Visibility::Public;
-        workList.push_back(member);
-        inserted = true;
       }
+      workList.push_back(member);
     }
-    return inserted;
   };
 
   // TODO: Consider Tags as well, but they should store HeapTypes instead of
diff --git a/test/lit/passes/type-refining.wast b/test/lit/passes/type-refining.wast
index d045dbc1a..91f61bc15 100644
--- a/test/lit/passes/type-refining.wast
+++ b/test/lit/passes/type-refining.wast
@@ -1519,3 +1519,58 @@
     )
   )
 )
+
+;; Regression test for a bug (#7103) in which the set of public types was
+;; computed incorrectly, leading to an assertion failure.
+(module
+ ;; CHECK:      (type $1 (sub (struct (field (mut (ref null $1))))))
+ (type $1 (sub (struct (field (mut (ref null $1))))))
+ (rec
+  ;; CHECK:      (rec
+  ;; CHECK-NEXT:  (type $7 (sub (struct (field (ref $1)))))
+
+  ;; CHECK:       (type $8 (sub (func)))
+
+  ;; CHECK:      (type $3 (func (result (ref null $8))))
+
+  ;; CHECK:      (rec
+  ;; CHECK-NEXT:  (type $5 (sub (struct (field (ref $3)))))
+  (type $5 (sub (struct (field (ref func)))))
+  ;; CHECK:       (type $6 (sub $1 (struct (field (mut (ref null $1))))))
+  (type $6 (sub $1 (struct (field (mut (ref null $1))))))
+ )
+ (rec
+  (type $7 (sub (struct (field (ref $1)))))
+  (type $8 (sub (func)))
+ )
+ ;; CHECK:       (type $9 (sub $6 (struct (field (mut (ref null $1))))))
+ (type $9 (sub $6 (struct (field (mut (ref null $1))))))
+
+ ;; CHECK:      (elem declare func $1)
+
+ ;; CHECK:      (export "func" (func $1))
+ (export "func" (func $1))
+
+ ;; CHECK:      (func $1 (type $3) (result (ref null $8))
+ ;; CHECK-NEXT:  (local $l (ref $9))
+ ;; CHECK-NEXT:  (drop
+ ;; CHECK-NEXT:   (struct.get $5 0
+ ;; CHECK-NEXT:    (struct.new $5
+ ;; CHECK-NEXT:     (ref.func $1)
+ ;; CHECK-NEXT:    )
+ ;; CHECK-NEXT:   )
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT:  (ref.null nofunc)
+ ;; CHECK-NEXT: )
+ (func $1 (result (ref null $8))
+  (local $l (ref $9))
+  (drop
+   (struct.get $5 0
+    (struct.new $5
+     (ref.func $1)
+    )
+   )
+  )
+  (ref.null $8)
+ )
+)