[Wasm GC] Do not inline a function with an RTT parameter (#3808)

Inlined parameters become locals, and rtts cannot be handled as locals, unlike
non-nullable values which we can at least fix up. So do not inline functions with
rtt params.

2 files changed, 13 insertions, 2 deletions

diff --git a/src/ir/type-updating.cpp b/src/ir/type-updating.cpp
index 312540b53..d909bffb4 100644
--- a/src/ir/type-updating.cpp
+++ b/src/ir/type-updating.cpp
@@ -21,7 +21,13 @@ namespace wasm {
 
 namespace TypeUpdating {
 
-void handleNonNullableLocals(Function* func, Module& wasm) {
+bool canHandleAsLocal(Type type) {
+  // Defaultable types are always ok. For non-nullable types, we can handle them
+  // using defaultable ones + ref.as_non_nulls.
+  return type.isDefaultable() || type.isRef();
+}
+
+void handleNonDefaultableLocals(Function* func, Module& wasm) {
   // Check if this is an issue.
   bool hasNonNullable = false;
   for (auto type : func->vars) {
diff --git a/src/ir/type-updating.h b/src/ir/type-updating.h
index fc438d53e..c9e4f4f0b 100644
--- a/src/ir/type-updating.h
+++ b/src/ir/type-updating.h
@@ -307,10 +307,15 @@ struct TypeUpdater
 
 namespace TypeUpdating {
 
+// Checks whether a type is valid as a local, or whether
+// handleNonDefaultableLocals() can handle it if not.
+bool canHandleAsLocal(Type type);
+
 // Finds non-nullable locals, which are currently not supported, and handles
 // them. Atm this turns them into nullable ones, and adds ref.as_non_null on
 // their uses (which keeps the type of the users identical).
-void handleNonNullableLocals(Function* func, Module& wasm);
+// This may also handle other types of nondefaultable locals in the future.
+void handleNonDefaultableLocals(Function* func, Module& wasm);
 
 } // namespace TypeUpdating