[Wasm GC] DeadArgumentElimination: Do not refine return types of tail callees (#4025)

If there is a tail call, we can't change the return type of the function, as it
must match in the functions doing a tail call of it.

1 files changed, 9 insertions, 3 deletions

diff --git a/src/passes/DeadArgumentElimination.cpp b/src/passes/DeadArgumentElimination.cpp
index ddba3bee8..7ba1be9f8 100644
--- a/src/passes/DeadArgumentElimination.cpp
+++ b/src/passes/DeadArgumentElimination.cpp
@@ -327,8 +327,14 @@ struct DAE : public Pass {
       // affect whether an argument is used or not, it just refines the type
       // where possible.
       refineArgumentTypes(func, calls, module);
-      // Refine return types as well.
-      if (refineReturnTypes(func, calls, module)) {
+      // Refine return types as well, if we can. We cannot do so if this
+      // function is tail-called, because then the return type must match that
+      // of the function doing a tail call of it - we cannot change just one of
+      // them.
+      //
+      // TODO: Try to optimize in a more holistic manner, see the TODO in
+      //       refineReturnTypes() about missing a global optimum.
+      if (!tailCallees.count(name) && refineReturnTypes(func, calls, module)) {
         refinedReturnTypes = true;
       }
       // Check if all calls pass the same constant for a particular argument.
@@ -434,7 +440,7 @@ struct DAE : public Pass {
         if (infoMap[name].hasTailCalls) {
           continue;
         }
-        if (tailCallees.find(name) != tailCallees.end()) {
+        if (tailCallees.count(name)) {
           continue;
         }
         auto iter = allCalls.find(name);