[GC] Do not crash on unreasonable GC array allocations in interpreter; trap (#3559)

The spec does not mention traps here, but this is like a JS VM trapping on
OOM - a runtime limitation is reached.

As these are not specced traps, I did not add them to effects.h. Note how
as a result the optimizer happily optimizes into a nop an unused allocation of an
array of size unsigned(-1), which is the behavior we want.

2 files changed, 19 insertions, 4 deletions

diff --git a/test/passes/Oz_fuzz-exec_all-features.txt b/test/passes/Oz_fuzz-exec_all-features.txt
index 0f90a6ef7..5d00a3db1 100644
--- a/test/passes/Oz_fuzz-exec_all-features.txt
+++ b/test/passes/Oz_fuzz-exec_all-features.txt
@@ -25,13 +25,15 @@
 [LoggingExternalInterface logging 0]
 [fuzz-exec] calling br_on_data
 [LoggingExternalInterface logging 1]
-[fuzz-exec] calling $rtt-and-cast-on-func
+[fuzz-exec] calling rtt-and-cast-on-func
 [LoggingExternalInterface logging 0]
 [LoggingExternalInterface logging 1]
 [LoggingExternalInterface logging 2]
 [LoggingExternalInterface logging 1337]
 [LoggingExternalInterface logging 3]
 [trap cast error]
+[fuzz-exec] calling array-alloc-failure
+[trap allocation failure]
 (module
  (type $struct (struct (field (mut i32))))
  (type $void_func (func))
@@ -48,7 +50,8 @@
  (export "br_on_cast" (func $3))
  (export "cast-null-anyref-to-gc" (func $4))
  (export "br_on_data" (func $6))
- (export "$rtt-and-cast-on-func" (func $8))
+ (export "rtt-and-cast-on-func" (func $8))
+ (export "array-alloc-failure" (func $9))
  (func $0 (; has Stack IR ;)
   (local $0 (ref null $struct))
   (call $log
@@ -235,6 +238,9 @@
    (i32.const 4)
   )
  )
+ (func $9 (; has Stack IR ;)
+  (nop)
+ )
 )
 [fuzz-exec] calling structs
 [LoggingExternalInterface logging 0]
@@ -263,10 +269,11 @@
 [LoggingExternalInterface logging 0]
 [fuzz-exec] calling br_on_data
 [LoggingExternalInterface logging 1]
-[fuzz-exec] calling $rtt-and-cast-on-func
+[fuzz-exec] calling rtt-and-cast-on-func
 [LoggingExternalInterface logging 0]
 [LoggingExternalInterface logging 1]
 [LoggingExternalInterface logging 2]
 [LoggingExternalInterface logging 1337]
 [LoggingExternalInterface logging 3]
 [trap cast error]
+[fuzz-exec] calling array-alloc-failure
diff --git a/test/passes/Oz_fuzz-exec_all-features.wast b/test/passes/Oz_fuzz-exec_all-features.wast
index 34c9a224a..4092f6b34 100644
--- a/test/passes/Oz_fuzz-exec_all-features.wast
+++ b/test/passes/Oz_fuzz-exec_all-features.wast
@@ -209,7 +209,7 @@
  (func $a-void-func
   (call $log (i32.const 1337))
  )
- (func "$rtt-and-cast-on-func"
+ (func "rtt-and-cast-on-func"
   (call $log (i32.const 0))
   (drop
    (rtt.canon $void_func)
@@ -231,4 +231,12 @@
   ;; will never be reached
   (call $log (i32.const 4))
  )
+ (func "array-alloc-failure"
+  (drop
+   (array.new_default_with_rtt $bytes
+    (i32.const -1) ;; un-allocatable size (4GB * sizeof(Literal))
+    (rtt.canon $bytes)
+   )
+  )
+ )
 )