[GC] Support casts of function types (#3554)

I had completely missed that the spec allows ref.cast etc. of function types,
and not just data. Function types do not have an RTT, unlike GC data, but we
can still cast them. A function reference has the canonical RTT of the signature
for that type, so it's like a simplified case of the GC world, without a hierarchy
of RTTs.

As it turns out, our validation did not rule out rtt.canon of a function type,
nor ref.cast of one, so we unintentionally already had all the support for this
aside from the actual casting, which this PR adds.

The addition is mostly trivial, except that we now need a Module in the base
ExpressionRunner class, so that we can go from a function name to the actual
function. This PR refactors things to allow that.

1 files changed, 45 insertions, 27 deletions

diff --git a/src/wasm-interpreter.h b/src/wasm-interpreter.h
index 26764c304..ba6f5d6ee 100644
--- a/src/wasm-interpreter.h
+++ b/src/wasm-interpreter.h
@@ -157,6 +157,10 @@ public:
 template<typename SubType>
 class ExpressionRunner : public OverriddenVisitor<SubType, Flow> {
 protected:
+  // Optional module context to search for globals and called functions. NULL if
+  // we are not interested in any context.
+  Module* module = nullptr;
+
   // Maximum depth before giving up.
   Index maxDepth;
   Index depth = 0;
@@ -183,9 +187,11 @@ public:
   // Indicates no limit of maxDepth or maxLoopIterations.
   static const Index NO_LIMIT = 0;
 
-  ExpressionRunner(Index maxDepth = NO_LIMIT,
+  ExpressionRunner(Module* module = nullptr,
+                   Index maxDepth = NO_LIMIT,
                    Index maxLoopIterations = NO_LIMIT)
-    : maxDepth(maxDepth), maxLoopIterations(maxLoopIterations) {}
+    : module(module), maxDepth(maxDepth), maxLoopIterations(maxLoopIterations) {
+  }
 
   Flow visit(Expression* curr) {
     depth++;
@@ -210,6 +216,9 @@ public:
     return ret;
   }
 
+  // Gets the module this runner is operating on.
+  Module* getModule() { return module; }
+
   Flow visitBlock(Block* curr) {
     NOTE_ENTER("Block");
     // special-case Block, because Block nesting (in their first element) can be
@@ -1426,24 +1435,38 @@ public:
       cast.outcome = cast.Null;
       return cast;
     }
-    // The input may not be a struct or an array; for example it could be an
+    // The input may not be GC data or a function; for example it could be an
     // anyref of null (already handled above) or anything else (handled here,
     // but this is for future use as atm the binaryen interpreter cannot
     // represent external references).
-    if (!cast.originalRef.isData()) {
+    if (!cast.originalRef.isData() && !cast.originalRef.isFunction()) {
       cast.outcome = cast.Failure;
       return cast;
     }
-    auto gcData = cast.originalRef.getGCData();
-    auto refRtt = gcData->rtt;
-    auto intendedRtt = rtt.getSingleValue();
-    if (!refRtt.isSubRtt(intendedRtt)) {
-      cast.outcome = cast.Failure;
+    Literal seenRtt;
+    Literal intendedRtt = rtt.getSingleValue();
+    if (cast.originalRef.isFunction()) {
+      // Function casts are simple in that they have no RTT hierarchies; instead
+      // each reference has the canonical RTT for the signature.
+      // We must have a module in order to perform the cast, to get the type.
+      assert(module);
+      auto* func = module->getFunction(cast.originalRef.getFunc());
+      seenRtt = Literal(Type(Rtt(0, func->sig)));
+      cast.castRef =
+        Literal(func->name, Type(intendedRtt.type.getHeapType(), Nullable));
     } else {
-      cast.outcome = cast.Success;
+      // GC data store an RTT in each instance.
+      assert(cast.originalRef.isData());
+      auto gcData = cast.originalRef.getGCData();
+      seenRtt = gcData->rtt;
       cast.castRef =
         Literal(gcData, Type(intendedRtt.type.getHeapType(), Nullable));
     }
+    if (!seenRtt.isSubRtt(intendedRtt)) {
+      cast.outcome = cast.Failure;
+    } else {
+      cast.outcome = cast.Success;
+    }
     return cast;
   }
 
@@ -1788,10 +1811,6 @@ public:
   static const Index NO_LIMIT = 0;
 
 protected:
-  // Optional module context to search for globals and called functions. NULL if
-  // we are not interested in any context.
-  Module* module = nullptr;
-
   // Flags indicating special requirements. See FlagValues.
   Flags flags = FlagValues::DEFAULT;
 
@@ -1808,12 +1827,9 @@ public:
                            Flags flags,
                            Index maxDepth,
                            Index maxLoopIterations)
-    : ExpressionRunner<SubType>(maxDepth, maxLoopIterations), module(module),
+    : ExpressionRunner<SubType>(module, maxDepth, maxLoopIterations),
       flags(flags) {}
 
-  // Gets the module this runner is operating on.
-  Module* getModule() { return module; }
-
   // Sets a known local value to use.
   void setLocalValue(Index index, Literals& values) {
     assert(values.isConcrete());
@@ -1858,8 +1874,8 @@ public:
   Flow visitGlobalGet(GlobalGet* curr) {
     NOTE_ENTER("GlobalGet");
     NOTE_NAME(curr->name);
-    if (module != nullptr) {
-      auto* global = module->getGlobal(curr->name);
+    if (this->module != nullptr) {
+      auto* global = this->module->getGlobal(curr->name);
       // Check if the global has an immutable value anyway
       if (!global->imported() && !global->mutable_) {
         return ExpressionRunner<SubType>::visit(global->init);
@@ -1875,10 +1891,11 @@ public:
   Flow visitGlobalSet(GlobalSet* curr) {
     NOTE_ENTER("GlobalSet");
     NOTE_NAME(curr->name);
-    if (!(flags & FlagValues::PRESERVE_SIDEEFFECTS) && module != nullptr) {
+    if (!(flags & FlagValues::PRESERVE_SIDEEFFECTS) &&
+        this->module != nullptr) {
       // If we are evaluating and not replacing the expression, remember the
       // constant value set, if any, for subsequent gets.
-      auto* global = module->getGlobal(curr->name);
+      auto* global = this->module->getGlobal(curr->name);
       assert(global->mutable_);
       auto setFlow = ExpressionRunner<SubType>::visit(curr->value);
       if (!setFlow.breaking()) {
@@ -1895,8 +1912,8 @@ public:
     // when replacing as long as the function does not have any side effects.
     // Might yield something useful for simple functions like `clamp`, sometimes
     // even if arguments are only partially constant or not constant at all.
-    if ((flags & FlagValues::TRAVERSE_CALLS) != 0 && module != nullptr) {
-      auto* func = module->getFunction(curr->target);
+    if ((flags & FlagValues::TRAVERSE_CALLS) != 0 && this->module != nullptr) {
+      auto* func = this->module->getFunction(curr->target);
       if (!func->imported()) {
         if (func->sig.results.isConcrete()) {
           auto numOperands = curr->operands.size();
@@ -2023,7 +2040,8 @@ class InitializerExpressionRunner
 
 public:
   InitializerExpressionRunner(GlobalManager& globals, Index maxDepth)
-    : ExpressionRunner<InitializerExpressionRunner<GlobalManager>>(maxDepth),
+    : ExpressionRunner<InitializerExpressionRunner<GlobalManager>>(nullptr,
+                                                                   maxDepth),
       globals(globals) {}
 
   Flow visitGlobalGet(GlobalGet* curr) { return Flow(globals[curr->name]); }
@@ -2384,8 +2402,8 @@ private:
     RuntimeExpressionRunner(ModuleInstanceBase& instance,
                             FunctionScope& scope,
                             Index maxDepth)
-      : ExpressionRunner<RuntimeExpressionRunner>(maxDepth), instance(instance),
-        scope(scope) {}
+      : ExpressionRunner<RuntimeExpressionRunner>(&instance.wasm, maxDepth),
+        instance(instance), scope(scope) {}
 
     Flow visitCall(Call* curr) {
       NOTE_ENTER("Call");