[Wasm GC] Do not inline a function with an RTT parameter (#3808)

Inlined parameters become locals, and rtts cannot be handled as locals, unlike
non-nullable values which we can at least fix up. So do not inline functions with
rtt params.

9 files changed, 58 insertions, 8 deletions

diff --git a/src/ir/type-updating.cpp b/src/ir/type-updating.cpp
index 312540b53..d909bffb4 100644
--- a/src/ir/type-updating.cpp
+++ b/src/ir/type-updating.cpp
@@ -21,7 +21,13 @@ namespace wasm {
 
 namespace TypeUpdating {
 
-void handleNonNullableLocals(Function* func, Module& wasm) {
+bool canHandleAsLocal(Type type) {
+  // Defaultable types are always ok. For non-nullable types, we can handle them
+  // using defaultable ones + ref.as_non_nulls.
+  return type.isDefaultable() || type.isRef();
+}
+
+void handleNonDefaultableLocals(Function* func, Module& wasm) {
   // Check if this is an issue.
   bool hasNonNullable = false;
   for (auto type : func->vars) {
diff --git a/src/ir/type-updating.h b/src/ir/type-updating.h
index fc438d53e..c9e4f4f0b 100644
--- a/src/ir/type-updating.h
+++ b/src/ir/type-updating.h
@@ -307,10 +307,15 @@ struct TypeUpdater
 
 namespace TypeUpdating {
 
+// Checks whether a type is valid as a local, or whether
+// handleNonDefaultableLocals() can handle it if not.
+bool canHandleAsLocal(Type type);
+
 // Finds non-nullable locals, which are currently not supported, and handles
 // them. Atm this turns them into nullable ones, and adds ref.as_non_null on
 // their uses (which keeps the type of the users identical).
-void handleNonNullableLocals(Function* func, Module& wasm);
+// This may also handle other types of nondefaultable locals in the future.
+void handleNonDefaultableLocals(Function* func, Module& wasm);
 
 } // namespace TypeUpdating
 
diff --git a/src/passes/DeadArgumentElimination.cpp b/src/passes/DeadArgumentElimination.cpp
index 59337f75d..37aaace37 100644
--- a/src/passes/DeadArgumentElimination.cpp
+++ b/src/passes/DeadArgumentElimination.cpp
@@ -383,7 +383,7 @@ struct DAE : public Pass {
             // Wonderful, nothing stands in our way! Do it.
             // TODO: parallelize this?
             removeParameter(func, i, calls);
-            TypeUpdating::handleNonNullableLocals(func, *module);
+            TypeUpdating::handleNonDefaultableLocals(func, *module);
             changed.insert(func);
           }
         }
diff --git a/src/passes/Flatten.cpp b/src/passes/Flatten.cpp
index c8f731d64..298b6241e 100644
--- a/src/passes/Flatten.cpp
+++ b/src/passes/Flatten.cpp
@@ -337,7 +337,7 @@ struct Flatten
     // the body may have preludes
     curr->body = getPreludesWithExpression(originalBody, curr->body);
     // New locals we added may be non-nullable.
-    TypeUpdating::handleNonNullableLocals(curr, *getModule());
+    TypeUpdating::handleNonDefaultableLocals(curr, *getModule());
     // We cannot handle non-nullable tuples currently, see the comment at the
     // top of the file.
     for (auto type : curr->vars) {
diff --git a/src/passes/Inlining.cpp b/src/passes/Inlining.cpp
index 0e6f6abe4..78f9966e7 100644
--- a/src/passes/Inlining.cpp
+++ b/src/passes/Inlining.cpp
@@ -52,6 +52,7 @@ struct FunctionInfo {
   bool hasLoops;
   bool hasTryDelegate;
   bool usedGlobally; // in a table or export
+  bool uninlineable;
 
   FunctionInfo() {
     refs = 0;
@@ -60,10 +61,14 @@ struct FunctionInfo {
     hasLoops = false;
     hasTryDelegate = false;
     usedGlobally = false;
+    uninlineable = false;
   }
 
   // See pass.h for how defaults for these options were chosen.
   bool worthInlining(PassOptions& options) {
+    if (uninlineable) {
+      return false;
+    }
     // Until we have proper support for try-delegate, ignore such functions.
     // FIXME https://github.com/WebAssembly/binaryen/issues/3634
     if (hasTryDelegate) {
@@ -134,7 +139,17 @@ struct FunctionInfoScanner
   }
 
   void visitFunction(Function* curr) {
-    (*infos)[curr->name].size = Measurer::measure(curr->body);
+    auto& info = (*infos)[curr->name];
+
+    // We cannot inline a function if we cannot handle placing it in a local, as
+    // all params become locals.
+    for (auto param : curr->sig.params) {
+      if (!TypeUpdating::canHandleAsLocal(param)) {
+        info.uninlineable = true;
+      }
+    }
+
+    info.size = Measurer::measure(curr->body);
   }
 
 private:
@@ -298,7 +313,7 @@ doInlining(Module* module, Function* into, const InliningAction& action) {
     // Make the block reachable by adding a break to it
     block->list.push_back(builder.makeBreak(block->name));
   }
-  TypeUpdating::handleNonNullableLocals(into, *module);
+  TypeUpdating::handleNonDefaultableLocals(into, *module);
   return block;
 }
 
diff --git a/src/passes/SSAify.cpp b/src/passes/SSAify.cpp
index 843015936..9f330f7b3 100644
--- a/src/passes/SSAify.cpp
+++ b/src/passes/SSAify.cpp
@@ -100,7 +100,7 @@ struct SSAify : public Pass {
     // add prepends to function
     addPrepends();
     // Handle non-nullability in new locals we added.
-    TypeUpdating::handleNonNullableLocals(func, *module);
+    TypeUpdating::handleNonDefaultableLocals(func, *module);
   }
 
   void createNewIndexes(LocalGraph& graph) {
diff --git a/src/wasm/wasm-binary.cpp b/src/wasm/wasm-binary.cpp
index b7a04a324..ddb0ece90 100644
--- a/src/wasm/wasm-binary.cpp
+++ b/src/wasm/wasm-binary.cpp
@@ -2186,7 +2186,7 @@ void WasmBinaryBuilder::readFunctions() {
       }
     }
 
-    TypeUpdating::handleNonNullableLocals(func, wasm);
+    TypeUpdating::handleNonDefaultableLocals(func, wasm);
 
     std::swap(func->epilogLocation, debugLocation);
     currFunction = nullptr;
diff --git a/test/passes/inlining_all-features.txt b/test/passes/inlining_all-features.txt
index 8ecf9fa8e..3f9aa57ce 100644
--- a/test/passes/inlining_all-features.txt
+++ b/test/passes/inlining_all-features.txt
@@ -125,3 +125,16 @@
   )
  )
 )
+(module
+ (type $struct (struct ))
+ (type $none_=>_none (func))
+ (type $rtt_$struct_=>_none (func (param (rtt $struct))))
+ (func $0 (param $rtt (rtt $struct))
+  (nop)
+ )
+ (func $1
+  (call $0
+   (rtt.canon $struct)
+  )
+ )
+)
diff --git a/test/passes/inlining_all-features.wast b/test/passes/inlining_all-features.wast
index d11353aa4..0aa324e1a 100644
--- a/test/passes/inlining_all-features.wast
+++ b/test/passes/inlining_all-features.wast
@@ -101,3 +101,14 @@
   )
  )
 )
+;; never inline an rtt parameter, as those cannot be handled as locals
+(module
+ (type $struct (struct))
+ (func $0 (param $rtt (rtt $struct))
+ )
+ (func $1
+  (call $0
+   (rtt.canon $struct)
+  )
+ )
+)