Fix emitting of function reference types without GC (#5737)

We previously had logic to emit GC types used in the IR as their corresponding
top types when GC was not enabled (so e.g. nullfuncref would be emitted as
funcref), but the logic was not robust enough and non-null function references
were not properly emitted as funcref.

Refactor the relevant code to be more robust and future-proof, and add a test
demonstrating that the lowering works as intended.

1 files changed, 17 insertions, 14 deletions

diff --git a/src/wasm/wasm-binary.cpp b/src/wasm/wasm-binary.cpp
index 91309c906..1d0e99415 100644
--- a/src/wasm/wasm-binary.cpp
+++ b/src/wasm/wasm-binary.cpp
@@ -1393,6 +1393,18 @@ void WasmBinaryWriter::writeInlineBuffer(const char* data, size_t size) {
 
 void WasmBinaryWriter::writeType(Type type) {
   if (type.isRef()) {
+    // The only reference types allowed without GC are funcref and externref. We
+    // internally use more refined versions of those types, but we cannot emit
+    // those more refined types.
+    if (!wasm->features.hasGC()) {
+      if (Type::isSubType(type, Type(HeapType::func, Nullable))) {
+        o << S32LEB(BinaryConsts::EncodedType::funcref);
+        return;
+      }
+      assert(Type::isSubType(type, Type(HeapType::ext, Nullable)));
+      o << S32LEB(BinaryConsts::EncodedType::externref);
+      return;
+    }
     auto heapType = type.getHeapType();
     if (heapType.isBasic() && type.isNullable()) {
       switch (heapType.getBasic()) {
@@ -1433,20 +1445,10 @@ void WasmBinaryWriter::writeType(Type type) {
           o << S32LEB(BinaryConsts::EncodedType::nullref);
           return;
         case HeapType::noext:
-          // See comment on writeHeapType.
-          if (!wasm->features.hasGC()) {
-            o << S32LEB(BinaryConsts::EncodedType::externref);
-          } else {
-            o << S32LEB(BinaryConsts::EncodedType::nullexternref);
-          }
+          o << S32LEB(BinaryConsts::EncodedType::nullexternref);
           return;
         case HeapType::nofunc:
-          // See comment on writeHeapType.
-          if (!wasm->features.hasGC()) {
-            o << S32LEB(BinaryConsts::EncodedType::funcref);
-          } else {
-            o << S32LEB(BinaryConsts::EncodedType::nullfuncref);
-          }
+          o << S32LEB(BinaryConsts::EncodedType::nullfuncref);
           return;
       }
     }
@@ -1491,9 +1493,10 @@ void WasmBinaryWriter::writeHeapType(HeapType type) {
   // only actually valid with GC enabled. When GC is not enabled, emit the
   // corresponding valid top types instead.
   if (!wasm->features.hasGC()) {
-    if (type == HeapType::nofunc || type.isSignature()) {
+    if (HeapType::isSubType(type, HeapType::func)) {
       type = HeapType::func;
-    } else if (type == HeapType::noext) {
+    } else {
+      assert(HeapType::isSubType(type, HeapType::ext));
       type = HeapType::ext;
     }
   }