Fix Vacuuming of code leading up to an infinite loop (#6632)

We had that logic right in other places, but the specific part of Vacuum that
looks at code that leads up to an unreachable did not check for infinite loops,
so it could remove them.

2 files changed, 46 insertions, 26 deletions

diff --git a/src/passes/Vacuum.cpp b/src/passes/Vacuum.cpp
index 41a8e02da..4a4963291 100644
--- a/src/passes/Vacuum.cpp
+++ b/src/passes/Vacuum.cpp
@@ -158,15 +158,16 @@ struct Vacuum : public WalkerPass<ExpressionStackWalker<Vacuum>> {
           continue;
         }
 
-        // Check if we may no longer be heading to a trap. Two situations count
-        // here: Control flow might branch, or we might call (since a call might
-        // reach an import; see notes on that in pass.h:trapsNeverHappen).
+        // Check if we may no longer be heading to a trap. We can only optimize
+        // if the trap will actually be reached. Two situations can prevent that
+        // here: Control flow might branch away, or we might hang (which can
+        // happen in a call or a loop).
         //
         // We also cannot remove a pop as it is necessary for structural
         // reasons.
         EffectAnalyzer effects(getPassOptions(), *getModule(), list[i]);
         if (effects.transfersControlFlow() || effects.calls ||
-            effects.danglingPop) {
+            effects.mayNotReturn || effects.danglingPop) {
           headingToTrap = false;
           continue;
         }
diff --git a/test/lit/passes/vacuum-tnh.wast b/test/lit/passes/vacuum-tnh.wast
index f74dc3c87..8a93a720f 100644
--- a/test/lit/passes/vacuum-tnh.wast
+++ b/test/lit/passes/vacuum-tnh.wast
@@ -117,10 +117,10 @@
   )
 
   ;; A helper function for the above, that returns nothing.
-  ;; YESTNH:      (func $return-nothing (type $1)
+  ;; YESTNH:      (func $return-nothing (type $0)
   ;; YESTNH-NEXT:  (nop)
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $return-nothing (type $1)
+  ;; NO_TNH:      (func $return-nothing (type $0)
   ;; NO_TNH-NEXT:  (nop)
   ;; NO_TNH-NEXT: )
   (func $return-nothing)
@@ -179,10 +179,10 @@
     (local.get $y)
   )
 
-  ;; YESTNH:      (func $toplevel (type $1)
+  ;; YESTNH:      (func $toplevel (type $0)
   ;; YESTNH-NEXT:  (nop)
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $toplevel (type $1)
+  ;; NO_TNH:      (func $toplevel (type $0)
   ;; NO_TNH-NEXT:  (unreachable)
   ;; NO_TNH-NEXT: )
   (func $toplevel
@@ -191,7 +191,7 @@
     (unreachable)
   )
 
-  ;; YESTNH:      (func $drop-loop (type $1)
+  ;; YESTNH:      (func $drop-loop (type $0)
   ;; YESTNH-NEXT:  (drop
   ;; YESTNH-NEXT:   (loop $loop (result i32)
   ;; YESTNH-NEXT:    (br_if $loop
@@ -201,7 +201,7 @@
   ;; YESTNH-NEXT:   )
   ;; YESTNH-NEXT:  )
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $drop-loop (type $1)
+  ;; NO_TNH:      (func $drop-loop (type $0)
   ;; NO_TNH-NEXT:  (drop
   ;; NO_TNH-NEXT:   (loop $loop (result i32)
   ;; NO_TNH-NEXT:    (br_if $loop
@@ -224,7 +224,7 @@
     )
   )
 
-  ;; YESTNH:      (func $loop-effects (type $1)
+  ;; YESTNH:      (func $loop-effects (type $0)
   ;; YESTNH-NEXT:  (drop
   ;; YESTNH-NEXT:   (loop $loop (result i32)
   ;; YESTNH-NEXT:    (drop
@@ -239,7 +239,7 @@
   ;; YESTNH-NEXT:   )
   ;; YESTNH-NEXT:  )
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $loop-effects (type $1)
+  ;; NO_TNH:      (func $loop-effects (type $0)
   ;; NO_TNH-NEXT:  (drop
   ;; NO_TNH-NEXT:   (loop $loop (result i32)
   ;; NO_TNH-NEXT:    (drop
@@ -272,7 +272,7 @@
     )
   )
 
-  ;; YESTNH:      (func $if-unreachable (type $0) (param $p i32)
+  ;; YESTNH:      (func $if-unreachable (type $1) (param $p i32)
   ;; YESTNH-NEXT:  (drop
   ;; YESTNH-NEXT:   (local.get $p)
   ;; YESTNH-NEXT:  )
@@ -294,7 +294,7 @@
   ;; YESTNH-NEXT:   )
   ;; YESTNH-NEXT:  )
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $if-unreachable (type $0) (param $p i32)
+  ;; NO_TNH:      (func $if-unreachable (type $1) (param $p i32)
   ;; NO_TNH-NEXT:  (if
   ;; NO_TNH-NEXT:   (local.get $p)
   ;; NO_TNH-NEXT:   (then
@@ -415,7 +415,7 @@
     )
   )
 
-  ;; YESTNH:      (func $block-unreachable (type $0) (param $p i32)
+  ;; YESTNH:      (func $block-unreachable (type $1) (param $p i32)
   ;; YESTNH-NEXT:  (if
   ;; YESTNH-NEXT:   (local.get $p)
   ;; YESTNH-NEXT:   (then
@@ -433,7 +433,7 @@
   ;; YESTNH-NEXT:   )
   ;; YESTNH-NEXT:  )
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $block-unreachable (type $0) (param $p i32)
+  ;; NO_TNH:      (func $block-unreachable (type $1) (param $p i32)
   ;; NO_TNH-NEXT:  (if
   ;; NO_TNH-NEXT:   (local.get $p)
   ;; NO_TNH-NEXT:   (then
@@ -482,7 +482,7 @@
     )
   )
 
-  ;; YESTNH:      (func $block-unreachable-named (type $0) (param $p i32)
+  ;; YESTNH:      (func $block-unreachable-named (type $1) (param $p i32)
   ;; YESTNH-NEXT:  (if
   ;; YESTNH-NEXT:   (local.get $p)
   ;; YESTNH-NEXT:   (then
@@ -499,7 +499,7 @@
   ;; YESTNH-NEXT:   )
   ;; YESTNH-NEXT:  )
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $block-unreachable-named (type $0) (param $p i32)
+  ;; NO_TNH:      (func $block-unreachable-named (type $1) (param $p i32)
   ;; NO_TNH-NEXT:  (if
   ;; NO_TNH-NEXT:   (local.get $p)
   ;; NO_TNH-NEXT:   (then
@@ -544,10 +544,10 @@
     )
   )
 
-  ;; YESTNH:      (func $block-unreachable-all (type $0) (param $p i32)
+  ;; YESTNH:      (func $block-unreachable-all (type $1) (param $p i32)
   ;; YESTNH-NEXT:  (nop)
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $block-unreachable-all (type $0) (param $p i32)
+  ;; NO_TNH:      (func $block-unreachable-all (type $1) (param $p i32)
   ;; NO_TNH-NEXT:  (if
   ;; NO_TNH-NEXT:   (local.get $p)
   ;; NO_TNH-NEXT:   (then
@@ -584,7 +584,7 @@
     )
   )
 
-  ;; YESTNH:      (func $block-unreachable-but-call (type $1)
+  ;; YESTNH:      (func $block-unreachable-but-call (type $0)
   ;; YESTNH-NEXT:  (i32.store
   ;; YESTNH-NEXT:   (i32.const 0)
   ;; YESTNH-NEXT:   (i32.const 1)
@@ -592,7 +592,7 @@
   ;; YESTNH-NEXT:  (call $block-unreachable-but-call)
   ;; YESTNH-NEXT:  (unreachable)
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $block-unreachable-but-call (type $1)
+  ;; NO_TNH:      (func $block-unreachable-but-call (type $0)
   ;; NO_TNH-NEXT:  (i32.store
   ;; NO_TNH-NEXT:   (i32.const 0)
   ;; NO_TNH-NEXT:   (i32.const 1)
@@ -620,7 +620,7 @@
     (unreachable)
   )
 
-  ;; YESTNH:      (func $catch-pop (type $1)
+  ;; YESTNH:      (func $catch-pop (type $0)
   ;; YESTNH-NEXT:  (try $try
   ;; YESTNH-NEXT:   (do
   ;; YESTNH-NEXT:    (call $catch-pop)
@@ -633,7 +633,7 @@
   ;; YESTNH-NEXT:   )
   ;; YESTNH-NEXT:  )
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $catch-pop (type $1)
+  ;; NO_TNH:      (func $catch-pop (type $0)
   ;; NO_TNH-NEXT:  (try $try
   ;; NO_TNH-NEXT:   (do
   ;; NO_TNH-NEXT:    (call $catch-pop)
@@ -673,7 +673,7 @@
     )
   )
 
-  ;; YESTNH:      (func $loop-unreachable (type $0) (param $p i32)
+  ;; YESTNH:      (func $loop-unreachable (type $1) (param $p i32)
   ;; YESTNH-NEXT:  (loop $loop
   ;; YESTNH-NEXT:   (i32.store
   ;; YESTNH-NEXT:    (i32.const 0)
@@ -688,7 +688,7 @@
   ;; YESTNH-NEXT:   (unreachable)
   ;; YESTNH-NEXT:  )
   ;; YESTNH-NEXT: )
-  ;; NO_TNH:      (func $loop-unreachable (type $0) (param $p i32)
+  ;; NO_TNH:      (func $loop-unreachable (type $1) (param $p i32)
   ;; NO_TNH-NEXT:  (loop $loop
   ;; NO_TNH-NEXT:   (i32.store
   ;; NO_TNH-NEXT:    (i32.const 0)
@@ -728,4 +728,23 @@
       (unreachable)
     )
   )
+
+  ;; YESTNH:      (func $unreached-infinite-loop (type $0)
+  ;; YESTNH-NEXT:  (loop $label$1
+  ;; YESTNH-NEXT:   (br $label$1)
+  ;; YESTNH-NEXT:  )
+  ;; YESTNH-NEXT: )
+  ;; NO_TNH:      (func $unreached-infinite-loop (type $0)
+  ;; NO_TNH-NEXT:  (loop $label$1
+  ;; NO_TNH-NEXT:   (br $label$1)
+  ;; NO_TNH-NEXT:  )
+  ;; NO_TNH-NEXT: )
+  (func $unreached-infinite-loop
+    ;; Code that reaches an unreachable can be removed in TNH mode, but an
+    ;; infinite loop may not reach it, so nothing can be removed here.
+    (loop $label$1
+      (br $label$1)
+    )
+    (unreachable)
+  )
 )