summaryrefslogtreecommitdiff
path: root/src/prebuilt
diff options
context:
space:
mode:
Diffstat (limited to 'src/prebuilt')
-rw-r--r--src/prebuilt/wasm2c_atomicops_source_declarations.cc136
-rw-r--r--src/prebuilt/wasm2c_simd_source_declarations.cc46
-rw-r--r--src/prebuilt/wasm2c_source_declarations.cc205
3 files changed, 246 insertions, 141 deletions
diff --git a/src/prebuilt/wasm2c_atomicops_source_declarations.cc b/src/prebuilt/wasm2c_atomicops_source_declarations.cc
index 8312d5d6..dc02591c 100644
--- a/src/prebuilt/wasm2c_atomicops_source_declarations.cc
+++ b/src/prebuilt/wasm2c_atomicops_source_declarations.cc
@@ -17,25 +17,25 @@ R"w2c_template( TRAP(UNALIGNED); \
R"w2c_template( }
)w2c_template"
R"w2c_template(
-#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \
+#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \
)w2c_template"
-R"w2c_template( static inline t3 name(wasm_rt_shared_memory_t* mem, u64 addr) { \
+R"w2c_template( static inline t3 name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr) { \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( t1 result; \
)w2c_template"
-R"w2c_template( t1 result; \
+R"w2c_template( result = atomic_load_explicit( \
)w2c_template"
-R"w2c_template( result = atomic_load_explicit( \
+R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \
)w2c_template"
-R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \
+R"w2c_template( memory_order_relaxed); \
)w2c_template"
-R"w2c_template( memory_order_relaxed); \
+R"w2c_template( force_read(result); \
)w2c_template"
-R"w2c_template( force_read(result); \
+R"w2c_template( return (t3)(t2)result; \
)w2c_template"
-R"w2c_template( return (t3)(t2)result; \
+R"w2c_template( } \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( DEF_MEM_CHECKS0(name, _shared_, t1, return, t3)
)w2c_template"
R"w2c_template(
DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT)
@@ -69,9 +69,9 @@ R"w2c_template(DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ
R"w2c_template(
#define DEFINE_SHARED_STORE(name, t1, t2) \
)w2c_template"
-R"w2c_template( static inline void name(wasm_rt_shared_memory_t* mem, u64 addr, t2 value) { \
+R"w2c_template( static inline void name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( t2 value) { \
)w2c_template"
R"w2c_template( t1 wrapped = (t1)value; \
)w2c_template"
@@ -81,7 +81,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr
)w2c_template"
R"w2c_template( memory_order_relaxed); \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( DEF_MEM_CHECKS1(name, _shared_, t1, , void, t2)
)w2c_template"
R"w2c_template(
DEFINE_SHARED_STORE(i32_store_shared, u32, u32)
@@ -105,9 +107,7 @@ R"w2c_template(DEFINE_SHARED_STORE(i64_store32_shared, u32, u64)
R"w2c_template(
#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \
)w2c_template"
-R"w2c_template( static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \
-)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \
)w2c_template"
R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \
)w2c_template"
@@ -121,9 +121,11 @@ R"w2c_template( return (t3)(t2)result;
)w2c_template"
R"w2c_template( } \
)w2c_template"
-R"w2c_template( static inline t3 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr) { \
+R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3) \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( static inline t3 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \
+)w2c_template"
+R"w2c_template( u64 addr) { \
)w2c_template"
R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \
)w2c_template"
@@ -137,7 +139,9 @@ R"w2c_template( force_read(result);
)w2c_template"
R"w2c_template( return (t3)(t2)result; \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( DEF_MEM_CHECKS0(name##_shared, _shared_, t1, return, t3)
)w2c_template"
R"w2c_template(
DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT)
@@ -157,9 +161,9 @@ R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ
R"w2c_template(
#define DEFINE_ATOMIC_STORE(name, t1, t2) \
)w2c_template"
-R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \
+R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( t2 value) { \
)w2c_template"
R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \
)w2c_template"
@@ -169,11 +173,11 @@ R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, siz
)w2c_template"
R"w2c_template( } \
)w2c_template"
-R"w2c_template( static inline void name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
+R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2) \
)w2c_template"
-R"w2c_template( t2 value) { \
+R"w2c_template( static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( u64 addr, t2 value) { \
)w2c_template"
R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \
)w2c_template"
@@ -183,7 +187,9 @@ R"w2c_template( atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof
)w2c_template"
R"w2c_template( wrapped); \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, , void, t2)
)w2c_template"
R"w2c_template(
DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32)
@@ -203,9 +209,9 @@ R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64)
R"w2c_template(
#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \
)w2c_template"
-R"w2c_template( static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \
+R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( t2 value) { \
)w2c_template"
R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \
)w2c_template"
@@ -223,11 +229,11 @@ R"w2c_template( return (t2)ret;
)w2c_template"
R"w2c_template( } \
)w2c_template"
-R"w2c_template( static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
+R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \
)w2c_template"
-R"w2c_template( t2 value) { \
+R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( u64 addr, t2 value) { \
)w2c_template"
R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \
)w2c_template"
@@ -239,7 +245,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr
)w2c_template"
R"w2c_template( return (t2)ret; \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2)
)w2c_template"
R"w2c_template(
DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32)
@@ -319,9 +327,9 @@ R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64)
R"w2c_template(
#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \
)w2c_template"
-R"w2c_template( static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \
+R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( t2 value) { \
)w2c_template"
R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \
)w2c_template"
@@ -337,11 +345,11 @@ R"w2c_template( return (t2)ret;
)w2c_template"
R"w2c_template( } \
)w2c_template"
-R"w2c_template( static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
+R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \
)w2c_template"
-R"w2c_template( t2 value) { \
+R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( u64 addr, t2 value) { \
)w2c_template"
R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \
)w2c_template"
@@ -353,7 +361,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr
)w2c_template"
R"w2c_template( return (t2)ret; \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2)
)w2c_template"
R"w2c_template(
DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32)
@@ -371,57 +381,57 @@ R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64)
R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64)
)w2c_template"
R"w2c_template(
-#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \
+#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \
)w2c_template"
-R"w2c_template( static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected, \
+R"w2c_template( static inline t1 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( t1 replacement) { \
+R"w2c_template( t1 expected, t1 replacement) { \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t2); \
+R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \
)w2c_template"
-R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \
+R"w2c_template( t2 expected_wrapped = (t2)expected; \
)w2c_template"
-R"w2c_template( t2 expected_wrapped = (t2)expected; \
+R"w2c_template( t2 replacement_wrapped = (t2)replacement; \
)w2c_template"
-R"w2c_template( t2 replacement_wrapped = (t2)replacement; \
+R"w2c_template( t2 ret; \
)w2c_template"
-R"w2c_template( t2 ret; \
+R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \
)w2c_template"
-R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \
+R"w2c_template( if (ret == expected_wrapped) { \
)w2c_template"
-R"w2c_template( if (ret == expected_wrapped) { \
+R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \
)w2c_template"
-R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \
+R"w2c_template( sizeof(t2)); \
)w2c_template"
-R"w2c_template( sizeof(t2)); \
+R"w2c_template( } \
)w2c_template"
-R"w2c_template( } \
+R"w2c_template( return (t1)expected_wrapped; \
)w2c_template"
-R"w2c_template( return (t1)expected_wrapped; \
+R"w2c_template( } \
)w2c_template"
-R"w2c_template( } \
+R"w2c_template( DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \
)w2c_template"
-R"w2c_template( static inline t1 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
+R"w2c_template( static inline t1 name##_shared_unchecked( \
)w2c_template"
-R"w2c_template( t1 expected, t1 replacement) { \
+R"w2c_template( wasm_rt_shared_memory_t* mem, u64 addr, t1 expected, t1 replacement) { \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t2); \
+R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \
)w2c_template"
-R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \
+R"w2c_template( t2 expected_wrapped = (t2)expected; \
)w2c_template"
-R"w2c_template( t2 expected_wrapped = (t2)expected; \
+R"w2c_template( t2 replacement_wrapped = (t2)replacement; \
)w2c_template"
-R"w2c_template( t2 replacement_wrapped = (t2)replacement; \
+R"w2c_template( atomic_compare_exchange_strong( \
)w2c_template"
-R"w2c_template( atomic_compare_exchange_strong( \
+R"w2c_template( (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \
)w2c_template"
-R"w2c_template( (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \
+R"w2c_template( &expected_wrapped, replacement_wrapped); \
)w2c_template"
-R"w2c_template( &expected_wrapped, replacement_wrapped); \
+R"w2c_template( return (t1)expected_wrapped; \
)w2c_template"
-R"w2c_template( return (t1)expected_wrapped; \
+R"w2c_template( } \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( DEF_MEM_CHECKS2(name##_shared, _shared_, t2, return, t1, t1, t1)
)w2c_template"
R"w2c_template(
DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8);
diff --git a/src/prebuilt/wasm2c_simd_source_declarations.cc b/src/prebuilt/wasm2c_simd_source_declarations.cc
index 5b903b26..a43a0006 100644
--- a/src/prebuilt/wasm2c_simd_source_declarations.cc
+++ b/src/prebuilt/wasm2c_simd_source_declarations.cc
@@ -15,26 +15,26 @@ R"w2c_template(#endif
R"w2c_template(// TODO: equivalent constraint for ARM and other architectures
)w2c_template"
R"w2c_template(
-#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \
+#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \
)w2c_template"
-R"w2c_template( static inline v128 name(wasm_rt_memory_t* mem, u64 addr) { \
+R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t); \
+R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \
)w2c_template"
-R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \
+R"w2c_template( SIMD_FORCE_READ(result); \
)w2c_template"
-R"w2c_template( SIMD_FORCE_READ(result); \
+R"w2c_template( return result; \
)w2c_template"
-R"w2c_template( return result; \
+R"w2c_template( } \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( DEF_MEM_CHECKS0(name, _, t, return, v128);
)w2c_template"
R"w2c_template(
#define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \
)w2c_template"
-R"w2c_template( static inline v128 name(wasm_rt_memory_t* mem, u64 addr, v128 vec) { \
+R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t); \
+R"w2c_template( v128 vec) { \
)w2c_template"
R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane); \
)w2c_template"
@@ -42,29 +42,35 @@ R"w2c_template( SIMD_FORCE_READ(result);
)w2c_template"
R"w2c_template( return result; \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( DEF_MEM_CHECKS1(name, _, t, return, v128, v128);
)w2c_template"
R"w2c_template(
-#define DEFINE_SIMD_STORE(name, t) \
+#define DEFINE_SIMD_STORE(name, t) \
+)w2c_template"
+R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \
+R"w2c_template( v128 value) { \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t); \
+R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \
)w2c_template"
-R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \
+R"w2c_template( } \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128);
)w2c_template"
R"w2c_template(
-#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \
+#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \
+)w2c_template"
+R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \
+R"w2c_template( v128 value) { \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t); \
+R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \
)w2c_template"
-R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \
+R"w2c_template( } \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128);
)w2c_template"
R"w2c_template(
// clang-format off
diff --git a/src/prebuilt/wasm2c_source_declarations.cc b/src/prebuilt/wasm2c_source_declarations.cc
index 87529c40..557bf67b 100644
--- a/src/prebuilt/wasm2c_source_declarations.cc
+++ b/src/prebuilt/wasm2c_source_declarations.cc
@@ -165,15 +165,32 @@ R"w2c_template( (CHECK_CALL_INDIRECT(table, ft, x), \
R"w2c_template( DO_CALL_INDIRECT(table, t, x, __VA_ARGS__))
)w2c_template"
R"w2c_template(
-#ifdef SUPPORT_MEMORY64
+static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) {
)w2c_template"
-R"w2c_template(#define RANGE_CHECK(mem, offset, len) \
+R"w2c_template(#if __has_builtin(__builtin_add_overflow)
+)w2c_template"
+R"w2c_template( return __builtin_add_overflow(a, b, resptr);
+)w2c_template"
+R"w2c_template(#elif defined(_MSC_VER)
+)w2c_template"
+R"w2c_template( return _addcarry_u64(0, a, b, resptr);
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#error "Missing implementation of __builtin_add_overflow or _addcarry_u64"
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(}
+)w2c_template"
+R"w2c_template(
+#define RANGE_CHECK(mem, offset, len) \
)w2c_template"
R"w2c_template( do { \
)w2c_template"
R"w2c_template( uint64_t res; \
)w2c_template"
-R"w2c_template( if (__builtin_add_overflow(offset, len, &res)) \
+R"w2c_template( if (UNLIKELY(add_overflow(offset, len, &res))) \
)w2c_template"
R"w2c_template( TRAP(OOB); \
)w2c_template"
@@ -183,16 +200,6 @@ R"w2c_template( TRAP(OOB); \
)w2c_template"
R"w2c_template( } while (0);
)w2c_template"
-R"w2c_template(#else
-)w2c_template"
-R"w2c_template(#define RANGE_CHECK(mem, offset, len) \
-)w2c_template"
-R"w2c_template( if (UNLIKELY(offset + (uint64_t)len > mem->size)) \
-)w2c_template"
-R"w2c_template( TRAP(OOB);
-)w2c_template"
-R"w2c_template(#endif
-)w2c_template"
R"w2c_template(
#if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS
)w2c_template"
@@ -215,21 +222,40 @@ R"w2c_template(#define WASM_RT_CHECK_BASE(mem)
R"w2c_template(#endif
)w2c_template"
R"w2c_template(
-#if WASM_RT_MEMCHECK_GUARD_PAGES
+// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for
+)w2c_template"
+R"w2c_template(// default-page-size, 32-bit memories. It may do nothing at all
+)w2c_template"
+R"w2c_template(// (if hardware bounds-checking is enabled via guard pages)
)w2c_template"
-R"w2c_template(#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem);
+R"w2c_template(// or it may do a slightly faster RANGE_CHECK.
+)w2c_template"
+R"w2c_template(#if WASM_RT_MEMCHECK_GUARD_PAGES
+)w2c_template"
+R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem);
)w2c_template"
R"w2c_template(#else
)w2c_template"
-R"w2c_template(#define MEMCHECK(mem, a, t) \
+R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) \
+)w2c_template"
+R"w2c_template( WASM_RT_CHECK_BASE(mem); \
)w2c_template"
-R"w2c_template( WASM_RT_CHECK_BASE(mem); \
+R"w2c_template( if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \
)w2c_template"
-R"w2c_template( RANGE_CHECK(mem, a, sizeof(t))
+R"w2c_template( TRAP(OOB);
)w2c_template"
R"w2c_template(#endif
)w2c_template"
R"w2c_template(
+// MEMCHECK_GENERAL can be used for any memory
+)w2c_template"
+R"w2c_template(#define MEMCHECK_GENERAL(mem, a, t) \
+)w2c_template"
+R"w2c_template( WASM_RT_CHECK_BASE(mem); \
+)w2c_template"
+R"w2c_template( RANGE_CHECK(mem, a, sizeof(t));
+)w2c_template"
+R"w2c_template(
#ifdef __GNUC__
)w2c_template"
R"w2c_template(#define FORCE_READ_INT(var) __asm__("" ::"r"(var));
@@ -299,30 +325,103 @@ R"w2c_template( load_data(MEM_ADDR(&m, o, s), i, s); \
R"w2c_template( } while (0)
)w2c_template"
R"w2c_template(
-#define DEFINE_LOAD(name, t1, t2, t3, force_read) \
+#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \
+)w2c_template"
+R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
+)w2c_template"
+R"w2c_template( u64 addr) { \
+)w2c_template"
+R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \
+)w2c_template"
+R"w2c_template( ret_kw name##_unchecked(mem, addr); \
+)w2c_template"
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \
+)w2c_template"
+R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \
+)w2c_template"
+R"w2c_template( ret_kw name##_unchecked(mem, addr); \
+)w2c_template"
+R"w2c_template( }
+)w2c_template"
+R"w2c_template(
+#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \
+)w2c_template"
+R"w2c_template( val_type1) \
+)w2c_template"
+R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
+)w2c_template"
+R"w2c_template( u64 addr, val_type1 val1) { \
+)w2c_template"
+R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \
+)w2c_template"
+R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \
+)w2c_template"
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \
+)w2c_template"
+R"w2c_template( val_type1 val1) { \
+)w2c_template"
+R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \
+)w2c_template"
+R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \
+)w2c_template"
+R"w2c_template( }
+)w2c_template"
+R"w2c_template(
+#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \
+)w2c_template"
+R"w2c_template( val_type1, val_type2) \
+)w2c_template"
+R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
+)w2c_template"
+R"w2c_template( u64 addr, val_type1 val1, \
+)w2c_template"
+R"w2c_template( val_type2 val2) { \
)w2c_template"
-R"w2c_template( static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \
+R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \
)w2c_template"
-R"w2c_template( t1 result; \
+R"w2c_template( } \
)w2c_template"
-R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \
+R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( sizeof(t1)); \
+R"w2c_template( val_type1 val1, val_type2 val2) { \
)w2c_template"
-R"w2c_template( force_read(result); \
+R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \
)w2c_template"
-R"w2c_template( return (t3)(t2)result; \
+R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \
)w2c_template"
R"w2c_template( }
)w2c_template"
R"w2c_template(
+#define DEFINE_LOAD(name, t1, t2, t3, force_read) \
+)w2c_template"
+R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \
+)w2c_template"
+R"w2c_template( t1 result; \
+)w2c_template"
+R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \
+)w2c_template"
+R"w2c_template( sizeof(t1)); \
+)w2c_template"
+R"w2c_template( force_read(result); \
+)w2c_template"
+R"w2c_template( return (t3)(t2)result; \
+)w2c_template"
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3)
+)w2c_template"
+R"w2c_template(
#define DEFINE_STORE(name, t1, t2) \
)w2c_template"
-R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \
+R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
-R"w2c_template( MEMCHECK(mem, addr, t1); \
+R"w2c_template( t2 value) { \
)w2c_template"
R"w2c_template( t1 wrapped = (t1)value; \
)w2c_template"
@@ -330,7 +429,9 @@ R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrappe
)w2c_template"
R"w2c_template( sizeof(t1)); \
)w2c_template"
-R"w2c_template( }
+R"w2c_template( } \
+)w2c_template"
+R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2)
)w2c_template"
R"w2c_template(
DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT)
@@ -976,7 +1077,7 @@ R"w2c_template( return sqrtf(x);
R"w2c_template(}
)w2c_template"
R"w2c_template(
-static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) {
+static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) {
)w2c_template"
R"w2c_template( RANGE_CHECK(mem, d, n);
)w2c_template"
@@ -989,11 +1090,11 @@ static inline void memory_copy(wasm_rt_memory_t* dest,
)w2c_template"
R"w2c_template( const wasm_rt_memory_t* src,
)w2c_template"
-R"w2c_template( u32 dest_addr,
+R"w2c_template( u64 dest_addr,
)w2c_template"
-R"w2c_template( u32 src_addr,
+R"w2c_template( u64 src_addr,
)w2c_template"
-R"w2c_template( u32 n) {
+R"w2c_template( u64 n) {
)w2c_template"
R"w2c_template( RANGE_CHECK(dest, dest_addr, n);
)w2c_template"
@@ -1010,7 +1111,7 @@ R"w2c_template( const u8* src,
)w2c_template"
R"w2c_template( u32 src_size,
)w2c_template"
-R"w2c_template( u32 dest_addr,
+R"w2c_template( u64 dest_addr,
)w2c_template"
R"w2c_template( u32 src_addr,
)w2c_template"
@@ -1046,7 +1147,7 @@ R"w2c_template( const wasm_elem_segment_exp
)w2c_template"
R"w2c_template( u32 src_size,
)w2c_template"
-R"w2c_template( u32 dest_addr,
+R"w2c_template( u64 dest_addr,
)w2c_template"
R"w2c_template( u32 src_addr,
)w2c_template"
@@ -1058,9 +1159,7 @@ R"w2c_template( if (UNLIKELY(src_addr + (uint64_t)n > src_size))
)w2c_template"
R"w2c_template( TRAP(OOB);
)w2c_template"
-R"w2c_template( if (UNLIKELY(dest_addr + (uint64_t)n > dest->size))
-)w2c_template"
-R"w2c_template( TRAP(OOB);
+R"w2c_template( RANGE_CHECK(dest, dest_addr, n);
)w2c_template"
R"w2c_template( for (u32 i = 0; i < n; i++) {
)w2c_template"
@@ -1107,7 +1206,7 @@ R"w2c_template(static inline void externref_table_init(wasm_rt_externref_table_t
)w2c_template"
R"w2c_template( u32 src_size,
)w2c_template"
-R"w2c_template( u32 dest_addr,
+R"w2c_template( u64 dest_addr,
)w2c_template"
R"w2c_template( u32 src_addr,
)w2c_template"
@@ -1117,9 +1216,7 @@ R"w2c_template( if (UNLIKELY(src_addr + (uint64_t)n > src_size))
)w2c_template"
R"w2c_template( TRAP(OOB);
)w2c_template"
-R"w2c_template( if (UNLIKELY(dest_addr + (uint64_t)n > dest->size))
-)w2c_template"
-R"w2c_template( TRAP(OOB);
+R"w2c_template( RANGE_CHECK(dest, dest_addr, n);
)w2c_template"
R"w2c_template( for (u32 i = 0; i < n; i++) {
)w2c_template"
@@ -1136,17 +1233,11 @@ R"w2c_template( static inline void type##_table_copy(wasm_rt_##type##_table_t*
)w2c_template"
R"w2c_template( const wasm_rt_##type##_table_t* src, \
)w2c_template"
-R"w2c_template( u32 dest_addr, u32 src_addr, u32 n) { \
+R"w2c_template( u64 dest_addr, u64 src_addr, u64 n) { \
)w2c_template"
-R"w2c_template( if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \
+R"w2c_template( RANGE_CHECK(dest, dest_addr, n); \
)w2c_template"
-R"w2c_template( TRAP(OOB); \
-)w2c_template"
-R"w2c_template( if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \
-)w2c_template"
-R"w2c_template( TRAP(OOB); \
-)w2c_template"
-R"w2c_template( \
+R"w2c_template( RANGE_CHECK(src, src_addr, n); \
)w2c_template"
R"w2c_template( memmove(dest->data + dest_addr, src->data + src_addr, \
)w2c_template"
@@ -1164,7 +1255,7 @@ R"w2c_template(
)w2c_template"
R"w2c_template( static inline wasm_rt_##type##_t type##_table_get( \
)w2c_template"
-R"w2c_template( const wasm_rt_##type##_table_t* table, u32 i) { \
+R"w2c_template( const wasm_rt_##type##_table_t* table, u64 i) { \
)w2c_template"
R"w2c_template( if (UNLIKELY(i >= table->size)) \
)w2c_template"
@@ -1184,7 +1275,7 @@ R"w2c_template(
)w2c_template"
R"w2c_template( static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \
)w2c_template"
-R"w2c_template( u32 i, const wasm_rt_##type##_t val) { \
+R"w2c_template( u64 i, const wasm_rt_##type##_t val) { \
)w2c_template"
R"w2c_template( if (UNLIKELY(i >= table->size)) \
)w2c_template"
@@ -1204,13 +1295,11 @@ R"w2c_template(
)w2c_template"
R"w2c_template( static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \
)w2c_template"
-R"w2c_template( u32 d, const wasm_rt_##type##_t val, \
-)w2c_template"
-R"w2c_template( u32 n) { \
+R"w2c_template( u64 d, const wasm_rt_##type##_t val, \
)w2c_template"
-R"w2c_template( if (UNLIKELY((uint64_t)d + n > table->size)) \
+R"w2c_template( u64 n) { \
)w2c_template"
-R"w2c_template( TRAP(OOB); \
+R"w2c_template( RANGE_CHECK(table, d, n); \
)w2c_template"
R"w2c_template( for (uint32_t i = d; i < d + n; i++) { \
)w2c_template"
generated by cgit v1.2.3 (git 2.39.1) at 2025-04-04 22:05:03 +0000