Fix regression in memory.fill due to Memory64 (#3176)

details: https://github.com/WebAssembly/binaryen/issues/3149

5 files changed, 31 insertions, 8 deletions

diff --git a/src/literal.h b/src/literal.h
index 4ae4f969a..ce245d940 100644
--- a/src/literal.h
+++ b/src/literal.h
@@ -213,6 +213,7 @@ public:
   }
 
   int64_t getInteger() const;
+  uint64_t getUnsigned() const;
   double getFloat() const;
   void getBits(uint8_t (&buf)[16]) const;
   // Equality checks for the type and the bits, so a nan float would
diff --git a/src/wasm-interpreter.h b/src/wasm-interpreter.h
index 5e98e0fec..4f3de9078 100644
--- a/src/wasm-interpreter.h
+++ b/src/wasm-interpreter.h
@@ -2346,7 +2346,7 @@ private:
         return flow;
       }
       Flow ret = Literal::makeFromUInt64(instance.memorySize, indexType);
-      uint64_t delta = flow.getSingleValue().getInteger();
+      uint64_t delta = flow.getSingleValue().getUnsigned();
       if (delta > uint32_t(-1) / Memory::kPageSize && indexType == Type::i32) {
         return fail;
       }
@@ -2384,7 +2384,7 @@ private:
       assert(curr->segment < instance.wasm.memory.segments.size());
       Memory::Segment& segment = instance.wasm.memory.segments[curr->segment];
 
-      Address destVal(dest.getSingleValue().getInteger());
+      Address destVal(dest.getSingleValue().getUnsigned());
       Address offsetVal(uint32_t(offset.getSingleValue().geti32()));
       Address sizeVal(uint32_t(size.getSingleValue().geti32()));
 
@@ -2428,9 +2428,9 @@ private:
       NOTE_EVAL1(dest);
       NOTE_EVAL1(source);
       NOTE_EVAL1(size);
-      Address destVal(dest.getSingleValue().getInteger());
-      Address sourceVal(source.getSingleValue().getInteger());
-      Address sizeVal(size.getSingleValue().getInteger());
+      Address destVal(dest.getSingleValue().getUnsigned());
+      Address sourceVal(source.getSingleValue().getUnsigned());
+      Address sizeVal(size.getSingleValue().getUnsigned());
 
       if (sourceVal + sizeVal > instance.memorySize * Memory::kPageSize ||
           destVal + sizeVal > instance.memorySize * Memory::kPageSize ||
@@ -2474,10 +2474,13 @@ private:
       NOTE_EVAL1(dest);
       NOTE_EVAL1(value);
       NOTE_EVAL1(size);
-      Address destVal(dest.getSingleValue().getInteger());
-      Address sizeVal(size.getSingleValue().getInteger());
+      Address destVal(dest.getSingleValue().getUnsigned());
+      Address sizeVal(size.getSingleValue().getUnsigned());
 
-      if (destVal + sizeVal > instance.memorySize * Memory::kPageSize) {
+      // FIXME: cheaper wrapping detection?
+      if (destVal > instance.memorySize * Memory::kPageSize ||
+          sizeVal > instance.memorySize * Memory::kPageSize ||
+          destVal + sizeVal > instance.memorySize * Memory::kPageSize) {
         trap("out of bounds memory access in memory.fill");
       }
       uint8_t val(value.getSingleValue().geti32());
diff --git a/src/wasm/literal.cpp b/src/wasm/literal.cpp
index eb28268fb..6b63d2ca5 100644
--- a/src/wasm/literal.cpp
+++ b/src/wasm/literal.cpp
@@ -196,6 +196,17 @@ int64_t Literal::getInteger() const {
   }
 }
 
+uint64_t Literal::getUnsigned() const {
+  switch (type.getBasic()) {
+    case Type::i32:
+      return static_cast<uint32_t>(i32);
+    case Type::i64:
+      return i64;
+    default:
+      abort();
+  }
+}
+
 double Literal::getFloat() const {
   switch (type.getBasic()) {
     case Type::f32:
diff --git a/test/spec/bulk-memory.wast b/test/spec/bulk-memory.wast
index 99485e66b..62e9cdb3a 100644
--- a/test/spec/bulk-memory.wast
+++ b/test/spec/bulk-memory.wast
@@ -39,6 +39,10 @@
 ;; Writing 0 bytes outside of memory limit is NOT allowed.
 (assert_trap (invoke "fill" (i32.const 0x10001) (i32.const 0) (i32.const 0)))
 
+;; Negative size
+(assert_trap (invoke "fill" (i32.const 15) (i32.const 14) (i32.const -2)))
+(assert_return (invoke "load8_u" (i32.const 15)) (i32.const 0))
+
 ;; memory.copy
 (module
   (memory 1 1)
diff --git a/test/spec/bulk-memory64.wast b/test/spec/bulk-memory64.wast
index 060191216..5d0090e73 100644
--- a/test/spec/bulk-memory64.wast
+++ b/test/spec/bulk-memory64.wast
@@ -39,6 +39,10 @@
 ;; Writing 0 bytes outside of memory limit is NOT allowed.
 (assert_trap (invoke "fill" (i64.const 0x10001) (i32.const 0) (i64.const 0)))
 
+;; Negative size
+(assert_trap (invoke "fill" (i64.const 15) (i32.const 14) (i64.const -2)))
+(assert_return (invoke "load8_u" (i64.const 15)) (i32.const 0))
+
 ;; memory.copy
 (module
   (memory i64 1 1)