[GC] dataref typing fixes (#3536)

dataref was not noted as isRef, and we also did not handle the LUB for it,
which caused validation errors. After these two fixes, it is possible to add a
testcase that goes through the optimizer.

View without whitespace as the LUB change has a lot of that.

3 files changed, 62 insertions, 18 deletions

diff --git a/src/wasm/wasm-type.cpp b/src/wasm/wasm-type.cpp
index 11f6cfbaf..f7a4fb13c 100644
--- a/src/wasm/wasm-type.cpp
+++ b/src/wasm/wasm-type.cpp
@@ -370,7 +370,7 @@ bool Type::isTuple() const {
 
 bool Type::isRef() const {
   if (isBasic()) {
-    return id >= funcref && id <= i31ref;
+    return id >= funcref && id <= dataref;
   } else {
     return getTypeInfo(*this)->isRef();
   }
@@ -396,7 +396,7 @@ bool Type::isData() const {
 
 bool Type::isNullable() const {
   if (isBasic()) {
-    return id >= funcref && id <= eqref; // except i31ref
+    return id >= funcref && id <= eqref; // except i31ref and dataref
   } else {
     return getTypeInfo(*this)->isNullable();
   }
@@ -662,23 +662,34 @@ Type Type::getLeastUpperBound(Type a, Type b) {
   if (a.size() != b.size()) {
     return Type::none; // a poison value that must not be consumed
   }
-  if (a.isRef()) {
-    if (b.isRef()) {
-      if (a.isFunction() && b.isFunction()) {
-        return Type::funcref;
-      }
-      if ((a == Type::i31ref && b == Type::eqref) ||
-          (a == Type::eqref && b == Type::i31ref)) {
-        return Type::eqref;
-      }
-      // The LUB of two different reference types is anyref, which may or may
-      // not be a valid type depending on whether the anyref feature is enabled.
-      // When anyref is disabled, it is possible for the finalization of invalid
-      // code to introduce a use of anyref via this function, but that is not a
-      // problem because it will be caught and rejected by validation.
-      return Type::anyref;
+  if (a.isRef() || b.isRef()) {
+    if (!a.isRef() || !b.isRef()) {
+      return Type::none;
+    }
+    auto handleNullability = [&](HeapType heapType) {
+      return Type(heapType,
+                  a.isNullable() || b.isNullable() ? Nullable : NonNullable);
+    };
+    auto aHeap = a.getHeapType();
+    auto bHeap = b.getHeapType();
+    if (aHeap.isFunction() && bHeap.isFunction()) {
+      return handleNullability(HeapType::func);
+    }
+    if (aHeap.isData() && bHeap.isData()) {
+      return handleNullability(HeapType::data);
+    }
+    if ((aHeap == HeapType::eq || aHeap == HeapType::i31 ||
+         aHeap == HeapType::data) &&
+        (bHeap == HeapType::eq || bHeap == HeapType::i31 ||
+         bHeap == HeapType::data)) {
+      return handleNullability(HeapType::eq);
     }
-    return Type::none;
+    // The LUB of two different reference types is anyref, which may or may
+    // not be a valid type depending on whether the anyref feature is enabled.
+    // When anyref is disabled, it is possible for the finalization of invalid
+    // code to introduce a use of anyref via this function, but that is not a
+    // problem because it will be caught and rejected by validation.
+    return Type::anyref;
   }
   if (a.isTuple()) {
     TypeList types;
diff --git a/test/passes/Oz_fuzz-exec_all-features.txt b/test/passes/Oz_fuzz-exec_all-features.txt
index 867179092..48c9ecb5e 100644
--- a/test/passes/Oz_fuzz-exec_all-features.txt
+++ b/test/passes/Oz_fuzz-exec_all-features.txt
@@ -23,18 +23,22 @@
 [trap unreachable]
 [fuzz-exec] calling cast-null-anyref-to-gc
 [LoggingExternalInterface logging 0]
+[fuzz-exec] calling br_on_data
+[LoggingExternalInterface logging 1]
 (module
  (type ${mut:i32} (struct (field (mut i32))))
  (type $none_=>_none (func))
  (type ${i32_f64} (struct (field i32) (field f64)))
  (type $[mut:i8] (array (mut i8)))
  (type $i32_=>_none (func (param i32)))
+ (type $anyref_=>_none (func (param anyref)))
  (import "fuzzing-support" "log-i32" (func $log (param i32)))
  (export "structs" (func $0))
  (export "arrays" (func $1))
  (export "rtts" (func $2))
  (export "br_on_cast" (func $3))
  (export "cast-null-anyref-to-gc" (func $4))
+ (export "br_on_data" (func $5))
  (func $0 (; has Stack IR ;)
   (local $0 (ref null ${mut:i32}))
   (call $log
@@ -168,6 +172,21 @@
    (i32.const 0)
   )
  )
+ (func $5 (; has Stack IR ;) (param $0 anyref)
+  (drop
+   (block $data (result (ref null data))
+    (drop
+     (br_on_data $data
+      (local.get $0)
+     )
+    )
+    (call $log
+     (i32.const 1)
+    )
+    (ref.null data)
+   )
+  )
+ )
 )
 [fuzz-exec] calling structs
 [LoggingExternalInterface logging 0]
@@ -194,3 +213,5 @@
 [trap unreachable]
 [fuzz-exec] calling cast-null-anyref-to-gc
 [LoggingExternalInterface logging 0]
+[fuzz-exec] calling br_on_data
+[LoggingExternalInterface logging 1]
diff --git a/test/passes/Oz_fuzz-exec_all-features.wast b/test/passes/Oz_fuzz-exec_all-features.wast
index fb3be9691..a33f02fb5 100644
--- a/test/passes/Oz_fuzz-exec_all-features.wast
+++ b/test/passes/Oz_fuzz-exec_all-features.wast
@@ -184,4 +184,16 @@
    )
   )
  )
+ (func "br_on_data" (param $x anyref)
+  (local $y anyref)
+  (drop
+   (block $data (result dataref)
+    (local.set $y
+     (br_on_data $data (local.get $x))
+    )
+    (call $log (i32.const 1))
+    (ref.null data)
+   )
+  )
+ )
 )