StackCheck: Check both under and overflow (#3091)

See emscripten-core/emscripten#9039 (comment)

The valid stack area is a region [A, B] in memory. Previously we just checked that
new stack positions S were S >= A, which prevented us from growing too much
(the stack grows down). But that only worked if the growth was small enough to not
overflow and become a big unsigned value. This PR makes us check the other way
too, which requires us to know where the stack starts out at.

This still supports the old way of just passing in the growth limit. We can remove it
after the roll.

In principle this can all be done on the LLVM side too after emscripten-core/emscripten#12057
but I'm not sure of the details there, and this is easy to fix here and get testing up
(which can help with later LLVM work).

This helps emscripten-core/emscripten#11860 by allowing us to clean up
some fastcomp-specific stuff in tests.

5 files changed, 216 insertions, 65 deletions

diff --git a/test/lld/basic_safe_stack.wat.out b/test/lld/basic_safe_stack.wat.out
index a72f3d215..9e6a916da 100644
--- a/test/lld/basic_safe_stack.wat.out
+++ b/test/lld/basic_safe_stack.wat.out
@@ -2,11 +2,13 @@
  (type $none_=>_none (func))
  (type $i32_=>_none (func (param i32)))
  (type $i32_=>_i32 (func (param i32) (result i32)))
+ (type $i32_i32_=>_none (func (param i32 i32)))
  (import "env" "__handle_stack_overflow" (func $__handle_stack_overflow))
  (memory $0 2)
  (table $0 1 1 funcref)
  (global $global$0 (mut i32) (i32.const 66112))
  (global $global$1 i32 (i32.const 568))
+ (global $__stack_base (mut i32) (i32.const 0))
  (global $__stack_limit (mut i32) (i32.const 0))
  (export "memory" (memory $0))
  (export "__wasm_call_ctors" (func $__wasm_call_ctors))
@@ -15,6 +17,7 @@
  (export "main" (func $main))
  (export "__data_end" (global $global$1))
  (export "__set_stack_limit" (func $__set_stack_limit))
+ (export "__set_stack_limits" (func $__set_stack_limits))
  (export "__growWasmMemory" (func $__growWasmMemory))
  (func $__wasm_call_ctors
   (nop)
@@ -22,11 +25,17 @@
  (func $stackRestore (param $0 i32)
   (local $1 i32)
   (if
-   (i32.lt_u
-    (local.tee $1
-     (local.get $0)
+   (i32.or
+    (i32.gt_u
+     (local.tee $1
+      (local.get $0)
+     )
+     (global.get $__stack_base)
+    )
+    (i32.lt_u
+     (local.get $1)
+     (global.get $__stack_limit)
     )
-    (global.get $__stack_limit)
    )
    (call $__handle_stack_overflow)
   )
@@ -40,19 +49,25 @@
   (local $3 i32)
   (block
    (if
-    (i32.lt_u
-     (local.tee $3
-      (local.tee $1
-       (i32.and
-        (i32.sub
-         (global.get $global$0)
-         (local.get $0)
+    (i32.or
+     (i32.gt_u
+      (local.tee $3
+       (local.tee $1
+        (i32.and
+         (i32.sub
+          (global.get $global$0)
+          (local.get $0)
+         )
+         (i32.const -16)
         )
-        (i32.const -16)
        )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $3)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (call $__handle_stack_overflow)
    )
@@ -70,6 +85,14 @@
    (local.get $0)
   )
  )
+ (func $__set_stack_limits (param $0 i32) (param $1 i32)
+  (global.set $__stack_base
+   (local.get $0)
+  )
+  (global.set $__stack_limit
+   (local.get $1)
+  )
+ )
  (func $__growWasmMemory (param $newSize i32) (result i32)
   (memory.grow
    (local.get $newSize)
@@ -95,6 +118,7 @@
     "stackAlloc",
     "main",
     "__set_stack_limit",
+    "__set_stack_limits",
     "__growWasmMemory"
   ],
   "namedGlobals": {
diff --git a/test/lld/recursive_safe_stack.wat.out b/test/lld/recursive_safe_stack.wat.out
index b84820e09..cceca00ab 100644
--- a/test/lld/recursive_safe_stack.wat.out
+++ b/test/lld/recursive_safe_stack.wat.out
@@ -2,6 +2,7 @@
  (type $i32_i32_=>_i32 (func (param i32 i32) (result i32)))
  (type $none_=>_none (func))
  (type $i32_=>_none (func (param i32)))
+ (type $i32_i32_=>_none (func (param i32 i32)))
  (type $none_=>_i32 (func (result i32)))
  (type $i32_=>_i32 (func (param i32) (result i32)))
  (import "env" "printf" (func $printf (param i32 i32) (result i32)))
@@ -12,6 +13,7 @@
  (global $global$0 (mut i32) (i32.const 66128))
  (global $global$1 i32 (i32.const 66128))
  (global $global$2 i32 (i32.const 587))
+ (global $__stack_base (mut i32) (i32.const 0))
  (global $__stack_limit (mut i32) (i32.const 0))
  (export "memory" (memory $0))
  (export "__wasm_call_ctors" (func $__wasm_call_ctors))
@@ -19,6 +21,7 @@
  (export "__data_end" (global $global$2))
  (export "main" (func $main))
  (export "__set_stack_limit" (func $__set_stack_limit))
+ (export "__set_stack_limits" (func $__set_stack_limits))
  (export "__growWasmMemory" (func $__growWasmMemory))
  (func $__wasm_call_ctors
   (nop)
@@ -29,16 +32,22 @@
   (local $4 i32)
   (block
    (if
-    (i32.lt_u
-     (local.tee $3
-      (local.tee $2
-       (i32.sub
-        (global.get $global$0)
-        (i32.const 16)
+    (i32.or
+     (i32.gt_u
+      (local.tee $3
+       (local.tee $2
+        (i32.sub
+         (global.get $global$0)
+         (i32.const 16)
+        )
        )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $3)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (call $__handle_stack_overflow)
    )
@@ -62,14 +71,20 @@
   )
   (block
    (if
-    (i32.lt_u
-     (local.tee $4
-      (i32.add
-       (local.get $2)
-       (i32.const 16)
+    (i32.or
+     (i32.gt_u
+      (local.tee $4
+       (i32.add
+        (local.get $2)
+        (i32.const 16)
+       )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $4)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (call $__handle_stack_overflow)
    )
@@ -88,16 +103,22 @@
   (local $2 i32)
   (block
    (if
-    (i32.lt_u
-     (local.tee $1
-      (local.tee $0
-       (i32.sub
-        (global.get $global$0)
-        (i32.const 16)
+    (i32.or
+     (i32.gt_u
+      (local.tee $1
+       (local.tee $0
+        (i32.sub
+         (global.get $global$0)
+         (i32.const 16)
+        )
        )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $1)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (call $__handle_stack_overflow)
    )
@@ -120,14 +141,20 @@
   )
   (block
    (if
-    (i32.lt_u
-     (local.tee $2
-      (i32.add
-       (local.get $0)
-       (i32.const 16)
+    (i32.or
+     (i32.gt_u
+      (local.tee $2
+       (i32.add
+        (local.get $0)
+        (i32.const 16)
+       )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $2)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (call $__handle_stack_overflow)
    )
@@ -145,6 +172,14 @@
    (local.get $0)
   )
  )
+ (func $__set_stack_limits (param $0 i32) (param $1 i32)
+  (global.set $__stack_base
+   (local.get $0)
+  )
+  (global.set $__stack_limit
+   (local.get $1)
+  )
+ )
  (func $__growWasmMemory (param $newSize i32) (result i32)
   (memory.grow
    (local.get $newSize)
@@ -169,6 +204,7 @@
     "__wasm_call_ctors",
     "main",
     "__set_stack_limit",
+    "__set_stack_limits",
     "__growWasmMemory"
   ],
   "namedGlobals": {
diff --git a/test/lld/safe_stack_standalone-wasm.wat.out b/test/lld/safe_stack_standalone-wasm.wat.out
index 48ea2be68..51b88661b 100644
--- a/test/lld/safe_stack_standalone-wasm.wat.out
+++ b/test/lld/safe_stack_standalone-wasm.wat.out
@@ -2,6 +2,7 @@
  (type $i32_i32_=>_i32 (func (param i32 i32) (result i32)))
  (type $none_=>_none (func))
  (type $i32_=>_none (func (param i32)))
+ (type $i32_i32_=>_none (func (param i32 i32)))
  (type $none_=>_i32 (func (result i32)))
  (type $i32_=>_i32 (func (param i32) (result i32)))
  (import "env" "printf" (func $printf (param i32 i32) (result i32)))
@@ -11,6 +12,7 @@
  (global $global$0 (mut i32) (i32.const 66128))
  (global $global$1 i32 (i32.const 66128))
  (global $global$2 i32 (i32.const 587))
+ (global $__stack_base (mut i32) (i32.const 0))
  (global $__stack_limit (mut i32) (i32.const 0))
  (export "memory" (memory $0))
  (export "__wasm_call_ctors" (func $__wasm_call_ctors))
@@ -18,6 +20,7 @@
  (export "__data_end" (global $global$2))
  (export "main" (func $main))
  (export "__set_stack_limit" (func $__set_stack_limit))
+ (export "__set_stack_limits" (func $__set_stack_limits))
  (export "__growWasmMemory" (func $__growWasmMemory))
  (func $__wasm_call_ctors
   (nop)
@@ -28,16 +31,22 @@
   (local $4 i32)
   (block
    (if
-    (i32.lt_u
-     (local.tee $3
-      (local.tee $2
-       (i32.sub
-        (global.get $global$0)
-        (i32.const 16)
+    (i32.or
+     (i32.gt_u
+      (local.tee $3
+       (local.tee $2
+        (i32.sub
+         (global.get $global$0)
+         (i32.const 16)
+        )
        )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $3)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (unreachable)
    )
@@ -61,14 +70,20 @@
   )
   (block
    (if
-    (i32.lt_u
-     (local.tee $4
-      (i32.add
-       (local.get $2)
-       (i32.const 16)
+    (i32.or
+     (i32.gt_u
+      (local.tee $4
+       (i32.add
+        (local.get $2)
+        (i32.const 16)
+       )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $4)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (unreachable)
    )
@@ -87,16 +102,22 @@
   (local $2 i32)
   (block
    (if
-    (i32.lt_u
-     (local.tee $1
-      (local.tee $0
-       (i32.sub
-        (global.get $global$0)
-        (i32.const 16)
+    (i32.or
+     (i32.gt_u
+      (local.tee $1
+       (local.tee $0
+        (i32.sub
+         (global.get $global$0)
+         (i32.const 16)
+        )
        )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $1)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (unreachable)
    )
@@ -119,14 +140,20 @@
   )
   (block
    (if
-    (i32.lt_u
-     (local.tee $2
-      (i32.add
-       (local.get $0)
-       (i32.const 16)
+    (i32.or
+     (i32.gt_u
+      (local.tee $2
+       (i32.add
+        (local.get $0)
+        (i32.const 16)
+       )
       )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $2)
+      (global.get $__stack_limit)
      )
-     (global.get $__stack_limit)
     )
     (unreachable)
    )
@@ -144,6 +171,14 @@
    (local.get $0)
   )
  )
+ (func $__set_stack_limits (param $0 i32) (param $1 i32)
+  (global.set $__stack_base
+   (local.get $0)
+  )
+  (global.set $__stack_limit
+   (local.get $1)
+  )
+ )
  (func $__growWasmMemory (param $newSize i32) (result i32)
   (memory.grow
    (local.get $newSize)
@@ -167,6 +202,7 @@
     "__wasm_call_ctors",
     "main",
     "__set_stack_limit",
+    "__set_stack_limits",
     "__growWasmMemory"
   ],
   "namedGlobals": {
diff --git a/test/passes/stack-check_enable-mutable-globals.txt b/test/passes/stack-check_enable-mutable-globals.txt
new file mode 100644
index 000000000..52ee091ef
--- /dev/null
+++ b/test/passes/stack-check_enable-mutable-globals.txt
@@ -0,0 +1,48 @@
+(module
+ (type $i32_=>_none (func (param i32)))
+ (type $i32_i32_=>_none (func (param i32 i32)))
+ (type $none_=>_i32 (func (result i32)))
+ (import "env" "__stack_pointer" (global $sp (mut i32)))
+ (global $__stack_base (mut i32) (i32.const 0))
+ (global $__stack_limit (mut i32) (i32.const 0))
+ (export "use_stack" (func $0))
+ (export "__set_stack_limit" (func $__set_stack_limit))
+ (export "__set_stack_limits" (func $__set_stack_limits))
+ (func $0 (result i32)
+  (local $0 i32)
+  (block
+   (if
+    (i32.or
+     (i32.gt_u
+      (local.tee $0
+       (i32.const 42)
+      )
+      (global.get $__stack_base)
+     )
+     (i32.lt_u
+      (local.get $0)
+      (global.get $__stack_limit)
+     )
+    )
+    (unreachable)
+   )
+   (global.set $sp
+    (local.get $0)
+   )
+  )
+  (global.get $sp)
+ )
+ (func $__set_stack_limit (param $0 i32)
+  (global.set $__stack_limit
+   (local.get $0)
+  )
+ )
+ (func $__set_stack_limits (param $0 i32) (param $1 i32)
+  (global.set $__stack_base
+   (local.get $0)
+  )
+  (global.set $__stack_limit
+   (local.get $1)
+  )
+ )
+)
diff --git a/test/passes/stack-check_enable-mutable-globals.wast b/test/passes/stack-check_enable-mutable-globals.wast
new file mode 100644
index 000000000..3028039fd
--- /dev/null
+++ b/test/passes/stack-check_enable-mutable-globals.wast
@@ -0,0 +1,7 @@
+(module
+ (import "env" "__stack_pointer" (global $sp (mut i32)))
+ (func "use_stack" (result i32)
+  (global.set $sp (i32.const 42))
+  (global.get $sp)
+ )
+)