Fix RTTs for RTT-less instructions (#4294)

Allocation and cast instructions without explicit RTTs should use the canonical
RTTs for the given types. Furthermore, the RTTs for nominal types should reflect
the static type hierarchy. Previously, however, we implemented allocations and
casts without RTTs using an alternative system that only used static types
rather than RTT values. This alternative system would work fine in a world
without first-class RTTs, but it did not properly allow mixing instructions that
use RTTs and instructions that do not use RTTs as intended by the M4 GC spec.

This PR fixes the issue by using canonical RTTs where appropriate and cleans up
the relevant casting code using std::variant.

3 files changed, 54 insertions, 40 deletions

diff --git a/test/lit/exec/rtts.wast b/test/lit/exec/rtts.wast
index 98304f8f2..7c090f180 100644
--- a/test/lit/exec/rtts.wast
+++ b/test/lit/exec/rtts.wast
@@ -3,8 +3,6 @@
 ;; Check that allocation and casting instructions with and without RTTs can be
 ;; mixed correctly.
 
-;; TODO: Fix the implementation to not fail an assertion on the functions commented out below.
-
 ;; RUN: wasm-opt %s -all --fuzz-exec-before -q --structural -o /dev/null 2>&1 \
 ;; RUN:   | filecheck %s --check-prefix=EQREC
 
@@ -52,11 +50,10 @@
     )
   )
 
-  ;; TODO: This should succeed in --nominal mode
   ;; EQREC:      [fuzz-exec] calling canon-sub
   ;; EQREC-NEXT: [LoggingExternalInterface logging 0]
   ;; NOMNL:      [fuzz-exec] calling canon-sub
-  ;; NOMNL-NEXT: [LoggingExternalInterface logging 0]
+  ;; NOMNL-NEXT: [LoggingExternalInterface logging 1]
   (func "canon-sub"
     (call $log
       (ref.test
@@ -66,19 +63,22 @@
     )
   )
 
-  ;; (func "canon-static"
-  ;;   (call $log
-  ;;     (ref.test_static $sub-struct
-  ;;       (call $make-sub-struct-canon)
-  ;;     )
-  ;;   )
-  ;; )
+  ;; EQREC:      [fuzz-exec] calling canon-static
+  ;; EQREC-NEXT: [LoggingExternalInterface logging 1]
+  ;; NOMNL:      [fuzz-exec] calling canon-static
+  ;; NOMNL-NEXT: [LoggingExternalInterface logging 1]
+  (func "canon-static"
+    (call $log
+      (ref.test_static $sub-struct
+        (call $make-sub-struct-canon)
+      )
+    )
+  )
 
-  ;; TODO: This should succeed in --nominal mode
   ;; EQREC:      [fuzz-exec] calling sub-canon
   ;; EQREC-NEXT: [LoggingExternalInterface logging 0]
   ;; NOMNL:      [fuzz-exec] calling sub-canon
-  ;; NOMNL-NEXT: [LoggingExternalInterface logging 0]
+  ;; NOMNL-NEXT: [LoggingExternalInterface logging 1]
   (func "sub-canon"
     (call $log
       (ref.test
@@ -101,31 +101,43 @@
     )
   )
 
-  ;; (func "sub-static"
-  ;;   (call $log
-  ;;     (ref.test_static $sub-struct
-  ;;       (call $make-sub-struct-sub)
-  ;;     )
-  ;;   )
-  ;; )
-
-  ;; (func "static-canon"
-  ;;   (call $log
-  ;;     (ref.test
-  ;;       (call $make-sub-struct-static)
-  ;;       (rtt.canon $sub-struct)
-  ;;     )
-  ;;   )
-  ;; )
-
-  ;; (func "static-sub"
-  ;;   (call $log
-  ;;     (ref.test
-  ;;       (call $make-sub-struct-static)
-  ;;       (global.get $sub-rtt)
-  ;;     )
-  ;;   )
-  ;; )
+  ;; EQREC:      [fuzz-exec] calling sub-static
+  ;; EQREC-NEXT: [LoggingExternalInterface logging 0]
+  ;; NOMNL:      [fuzz-exec] calling sub-static
+  ;; NOMNL-NEXT: [LoggingExternalInterface logging 1]
+  (func "sub-static"
+    (call $log
+      (ref.test_static $sub-struct
+        (call $make-sub-struct-sub)
+      )
+    )
+  )
+
+  ;; EQREC:      [fuzz-exec] calling static-canon
+  ;; EQREC-NEXT: [LoggingExternalInterface logging 1]
+  ;; NOMNL:      [fuzz-exec] calling static-canon
+  ;; NOMNL-NEXT: [LoggingExternalInterface logging 1]
+  (func "static-canon"
+    (call $log
+      (ref.test
+        (call $make-sub-struct-static)
+        (rtt.canon $sub-struct)
+      )
+    )
+  )
+
+  ;; EQREC:      [fuzz-exec] calling static-sub
+  ;; EQREC-NEXT: [LoggingExternalInterface logging 0]
+  ;; NOMNL:      [fuzz-exec] calling static-sub
+  ;; NOMNL-NEXT: [LoggingExternalInterface logging 1]
+  (func "static-sub"
+    (call $log
+      (ref.test
+        (call $make-sub-struct-static)
+        (global.get $sub-rtt)
+      )
+    )
+  )
 
   ;; EQREC:      [fuzz-exec] calling static-static
   ;; EQREC-NEXT: [LoggingExternalInterface logging 1]
diff --git a/test/passes/Oz_fuzz-exec_all-features.txt b/test/passes/Oz_fuzz-exec_all-features.txt
index 359cb83ed..a7c327d1e 100644
--- a/test/passes/Oz_fuzz-exec_all-features.txt
+++ b/test/passes/Oz_fuzz-exec_all-features.txt
@@ -79,7 +79,7 @@
 [LoggingExternalInterface logging 0]
 [LoggingExternalInterface logging 1]
 [LoggingExternalInterface logging 0]
-[LoggingExternalInterface logging 1]
+[LoggingExternalInterface logging 0]
 [fuzz-exec] calling static-br_on_cast
 [LoggingExternalInterface logging 3]
 [fuzz-exec] calling static-br_on_cast_fail
diff --git a/test/passes/Oz_fuzz-exec_all-features.wast b/test/passes/Oz_fuzz-exec_all-features.wast
index ea29f259a..7ab9116dd 100644
--- a/test/passes/Oz_fuzz-exec_all-features.wast
+++ b/test/passes/Oz_fuzz-exec_all-features.wast
@@ -574,7 +574,9 @@
     (struct.new_default $struct)
    )
   )
-  ;; Casting to a supertype works.
+  ;; Casting to a supertype does not work because the canonical RTT for the
+  ;; subtype is not a sub-rtt of the canonical RTT of the supertype in
+  ;; structural mode.
   (call $log
    (ref.test_static $struct
     (struct.new_default $extendedstruct)