[GC] dataref typing fixes (#3536)

dataref was not noted as isRef, and we also did not handle the LUB for it,
which caused validation errors. After these two fixes, it is possible to add a
testcase that goes through the optimizer.

View without whitespace as the LUB change has a lot of that.

1 files changed, 29 insertions, 18 deletions

diff --git a/src/wasm/wasm-type.cpp b/src/wasm/wasm-type.cpp
index 11f6cfbaf..f7a4fb13c 100644
--- a/src/wasm/wasm-type.cpp
+++ b/src/wasm/wasm-type.cpp
@@ -370,7 +370,7 @@ bool Type::isTuple() const {
 
 bool Type::isRef() const {
   if (isBasic()) {
-    return id >= funcref && id <= i31ref;
+    return id >= funcref && id <= dataref;
   } else {
     return getTypeInfo(*this)->isRef();
   }
@@ -396,7 +396,7 @@ bool Type::isData() const {
 
 bool Type::isNullable() const {
   if (isBasic()) {
-    return id >= funcref && id <= eqref; // except i31ref
+    return id >= funcref && id <= eqref; // except i31ref and dataref
   } else {
     return getTypeInfo(*this)->isNullable();
   }
@@ -662,23 +662,34 @@ Type Type::getLeastUpperBound(Type a, Type b) {
   if (a.size() != b.size()) {
     return Type::none; // a poison value that must not be consumed
   }
-  if (a.isRef()) {
-    if (b.isRef()) {
-      if (a.isFunction() && b.isFunction()) {
-        return Type::funcref;
-      }
-      if ((a == Type::i31ref && b == Type::eqref) ||
-          (a == Type::eqref && b == Type::i31ref)) {
-        return Type::eqref;
-      }
-      // The LUB of two different reference types is anyref, which may or may
-      // not be a valid type depending on whether the anyref feature is enabled.
-      // When anyref is disabled, it is possible for the finalization of invalid
-      // code to introduce a use of anyref via this function, but that is not a
-      // problem because it will be caught and rejected by validation.
-      return Type::anyref;
+  if (a.isRef() || b.isRef()) {
+    if (!a.isRef() || !b.isRef()) {
+      return Type::none;
+    }
+    auto handleNullability = [&](HeapType heapType) {
+      return Type(heapType,
+                  a.isNullable() || b.isNullable() ? Nullable : NonNullable);
+    };
+    auto aHeap = a.getHeapType();
+    auto bHeap = b.getHeapType();
+    if (aHeap.isFunction() && bHeap.isFunction()) {
+      return handleNullability(HeapType::func);
+    }
+    if (aHeap.isData() && bHeap.isData()) {
+      return handleNullability(HeapType::data);
+    }
+    if ((aHeap == HeapType::eq || aHeap == HeapType::i31 ||
+         aHeap == HeapType::data) &&
+        (bHeap == HeapType::eq || bHeap == HeapType::i31 ||
+         bHeap == HeapType::data)) {
+      return handleNullability(HeapType::eq);
     }
-    return Type::none;
+    // The LUB of two different reference types is anyref, which may or may
+    // not be a valid type depending on whether the anyref feature is enabled.
+    // When anyref is disabled, it is possible for the finalization of invalid
+    // code to introduce a use of anyref via this function, but that is not a
+    // problem because it will be caught and rejected by validation.
+    return Type::anyref;
   }
   if (a.isTuple()) {
     TypeList types;