[Wasm GC] Fix casting code in interpreter (#3873)

The logic there would construct the cast value separately for functions and data
(as we must), and then in an attempt to share code, would then check if the
cast succeed or not (and if not, do nothing with the cast value).

But this was wrong, as in some weird casts (like a struct to a function) we
cannot construct a valid cast value, and we error there. Instead, check if the
cast works first, once we know enough to do so, and only then construct the
cast value if so.

3 files changed, 39 insertions, 6 deletions

diff --git a/src/wasm-interpreter.h b/src/wasm-interpreter.h
index 008825632..4cb26d74a 100644
--- a/src/wasm-interpreter.h
+++ b/src/wasm-interpreter.h
@@ -1453,6 +1453,10 @@ public:
         return cast;
       }
       seenRtt = Literal(Type(Rtt(0, func->sig)));
+      if (!seenRtt.isSubRtt(intendedRtt)) {
+        cast.outcome = cast.Failure;
+        return cast;
+      }
       cast.castRef =
         Literal(func->name, Type(intendedRtt.type.getHeapType(), NonNullable));
     } else {
@@ -1460,14 +1464,14 @@ public:
       assert(cast.originalRef.isData());
       auto gcData = cast.originalRef.getGCData();
       seenRtt = gcData->rtt;
+      if (!seenRtt.isSubRtt(intendedRtt)) {
+        cast.outcome = cast.Failure;
+        return cast;
+      }
       cast.castRef =
         Literal(gcData, Type(intendedRtt.type.getHeapType(), NonNullable));
     }
-    if (!seenRtt.isSubRtt(intendedRtt)) {
-      cast.outcome = cast.Failure;
-    } else {
-      cast.outcome = cast.Success;
-    }
+    cast.outcome = cast.Success;
     return cast;
   }
 
diff --git a/test/passes/Oz_fuzz-exec_all-features.txt b/test/passes/Oz_fuzz-exec_all-features.txt
index 6a0c63431..cf3335dce 100644
--- a/test/passes/Oz_fuzz-exec_all-features.txt
+++ b/test/passes/Oz_fuzz-exec_all-features.txt
@@ -36,6 +36,8 @@
 [host limit allocation failure]
 [fuzz-exec] calling init-array-packed
 [fuzz-exec] note result: init-array-packed => 213
+[fuzz-exec] calling cast-func-to-struct
+[trap cast error]
 (module
  (type $struct (struct (field (mut i32))))
  (type $void_func (func))
@@ -44,8 +46,9 @@
  (type $int_func (func (result i32)))
  (type $i32_=>_none (func (param i32)))
  (type $anyref_=>_none (func (param anyref)))
+ (type $eqref_=>_none (func (param eqref)))
  (import "fuzzing-support" "log-i32" (func $log (param i32)))
- (elem declare func $a-void-func)
+ (elem declare func $a-void-func $call-target)
  (export "structs" (func $0))
  (export "arrays" (func $1))
  (export "rtts" (func $2))
@@ -55,6 +58,7 @@
  (export "rtt-and-cast-on-func" (func $8))
  (export "array-alloc-failure" (func $9))
  (export "init-array-packed" (func $10))
+ (export "cast-func-to-struct" (func $12))
  (func $0 (; has Stack IR ;)
   (local $0 (ref null $struct))
   (call $log
@@ -293,6 +297,17 @@
    (i32.const 10)
   )
  )
+ (func $call-target (; has Stack IR ;) (param $0 eqref)
+  (nop)
+ )
+ (func $12 (; has Stack IR ;)
+  (drop
+   (ref.cast
+    (ref.func $call-target)
+    (rtt.canon $struct)
+   )
+  )
+ )
 )
 [fuzz-exec] calling structs
 [LoggingExternalInterface logging 0]
@@ -331,6 +346,8 @@
 [fuzz-exec] calling array-alloc-failure
 [fuzz-exec] calling init-array-packed
 [fuzz-exec] note result: init-array-packed => 213
+[fuzz-exec] calling cast-func-to-struct
+[trap cast error]
 ignoring comparison of ExecutionResults!
 [fuzz-exec] calling foo
 [host limit allocation failure]
diff --git a/test/passes/Oz_fuzz-exec_all-features.wast b/test/passes/Oz_fuzz-exec_all-features.wast
index 0bd41ca47..32570852d 100644
--- a/test/passes/Oz_fuzz-exec_all-features.wast
+++ b/test/passes/Oz_fuzz-exec_all-features.wast
@@ -256,6 +256,18 @@
    (i32.const 10)
   )
  )
+ (func $call-target (param $0 eqref)
+  (nop)
+ )
+ (func "cast-func-to-struct"
+  (drop
+   ;; An impossible cast of a function to a struct, which should fail.
+   (ref.cast
+    (ref.func $call-target)
+    (rtt.canon $struct)
+   )
+  )
+ )
 )
 (module
  (type $[mut:i8] (array (mut i8)))