#include #ifndef WASM_RT_C11_AVAILABLE #error "C11 is required for Wasm threads and shared memory support" #endif #define ATOMIC_ALIGNMENT_CHECK(addr, t1) \ if (UNLIKELY(addr % sizeof(t1))) { \ TRAP(UNALIGNED); \ } #define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ static inline t3 name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr) { \ t1 result; \ result = atomic_load_explicit( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ memory_order_relaxed); \ force_read(result); \ return (t3)(t2)result; \ } \ DEF_MEM_CHECKS0(name, _shared_, t1, return, t3) DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load_shared, u64, u64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(f32_load_shared, f32, f32, f32, FORCE_READ_FLOAT) DEFINE_SHARED_LOAD(f64_load_shared, f64, f64, f64, FORCE_READ_FLOAT) DEFINE_SHARED_LOAD(i32_load8_s_shared, s8, s32, u32, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load8_s_shared, s8, s64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(i32_load8_u_shared, u8, u32, u32, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load8_u_shared, u8, u64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(i32_load16_s_shared, s16, s32, u32, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load16_s_shared, s16, s64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(i32_load16_u_shared, u16, u32, u32, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load16_u_shared, u16, u64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load32_s_shared, s32, s64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ_INT) #define DEFINE_SHARED_STORE(name, t1, t2) \ static inline void name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr, \ t2 value) { \ t1 wrapped = (t1)value; \ atomic_store_explicit( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped, \ memory_order_relaxed); \ } \ DEF_MEM_CHECKS1(name, _shared_, t1, , void, t2) DEFINE_SHARED_STORE(i32_store_shared, u32, u32) DEFINE_SHARED_STORE(i64_store_shared, u64, u64) DEFINE_SHARED_STORE(f32_store_shared, f32, f32) DEFINE_SHARED_STORE(f64_store_shared, f64, f64) DEFINE_SHARED_STORE(i32_store8_shared, u8, u32) DEFINE_SHARED_STORE(i32_store16_shared, u16, u32) DEFINE_SHARED_STORE(i64_store8_shared, u8, u64) DEFINE_SHARED_STORE(i64_store16_shared, u16, u64) DEFINE_SHARED_STORE(i64_store32_shared, u32, u64) #define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 result; \ wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ force_read(result); \ return (t3)(t2)result; \ } \ DEF_MEM_CHECKS0(name, _, t1, return, t3) \ static inline t3 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ u64 addr) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 result; \ result = \ atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \ force_read(result); \ return (t3)(t2)result; \ } \ DEF_MEM_CHECKS0(name##_shared, _shared_, t1, return, t3) DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load, u64, u64, u64, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i32_atomic_load8_u, u8, u32, u32, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load8_u, u8, u64, u64, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i32_atomic_load16_u, u16, u32, u32, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT) #define DEFINE_ATOMIC_STORE(name, t1, t2) \ static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ } \ DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ wrapped); \ } \ DEF_MEM_CHECKS1(name##_shared, _shared_, t1, , void, t2) DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32) DEFINE_ATOMIC_STORE(i64_atomic_store, u64, u64) DEFINE_ATOMIC_STORE(i32_atomic_store8, u8, u32) DEFINE_ATOMIC_STORE(i32_atomic_store16, u16, u32) DEFINE_ATOMIC_STORE(i64_atomic_store8, u8, u64) DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64) DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) #define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret; \ wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ ret = ret op wrapped; \ wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &ret, sizeof(t1)); \ return (t2)ret; \ } \ DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret = atomic_##opname( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ return (t2)ret; \ } \ DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw16_add_u, fetch_add, +, u16, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw_add, fetch_add, +, u32, u32) DEFINE_ATOMIC_RMW(i64_atomic_rmw8_add_u, fetch_add, +, u8, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw16_add_u, fetch_add, +, u16, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw32_add_u, fetch_add, +, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_add, fetch_add, +, u64, u64) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_sub_u, fetch_sub, -, u8, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw16_sub_u, fetch_sub, -, u16, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw_sub, fetch_sub, -, u32, u32) DEFINE_ATOMIC_RMW(i64_atomic_rmw8_sub_u, fetch_sub, -, u8, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw16_sub_u, fetch_sub, -, u16, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw32_sub_u, fetch_sub, -, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_sub, fetch_sub, -, u64, u64) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_and_u, fetch_and, &, u8, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw16_and_u, fetch_and, &, u16, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw_and, fetch_and, &, u32, u32) DEFINE_ATOMIC_RMW(i64_atomic_rmw8_and_u, fetch_and, &, u8, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw16_and_u, fetch_and, &, u16, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw32_and_u, fetch_and, &, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_and, fetch_and, &, u64, u64) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_or_u, fetch_or, |, u8, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw16_or_u, fetch_or, |, u16, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw_or, fetch_or, |, u32, u32) DEFINE_ATOMIC_RMW(i64_atomic_rmw8_or_u, fetch_or, |, u8, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw16_or_u, fetch_or, |, u16, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw32_or_u, fetch_or, |, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_or, fetch_or, |, u64, u64) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_xor_u, fetch_xor, ^, u8, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw16_xor_u, fetch_xor, ^, u16, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw_xor, fetch_xor, ^, u32, u32) DEFINE_ATOMIC_RMW(i64_atomic_rmw8_xor_u, fetch_xor, ^, u8, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xor_u, fetch_xor, ^, u16, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) #define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret; \ wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ return (t2)ret; \ } \ DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret = atomic_##opname( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ return (t2)ret; \ } \ DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32) DEFINE_ATOMIC_XCHG(i32_atomic_rmw16_xchg_u, exchange, u16, u32) DEFINE_ATOMIC_XCHG(i32_atomic_rmw_xchg, exchange, u32, u32) DEFINE_ATOMIC_XCHG(i64_atomic_rmw8_xchg_u, exchange, u8, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw16_xchg_u, exchange, u16, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64) #define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ static inline t1 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ t1 expected, t1 replacement) { \ ATOMIC_ALIGNMENT_CHECK(addr, t2); \ t2 expected_wrapped = (t2)expected; \ t2 replacement_wrapped = (t2)replacement; \ t2 ret; \ wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ if (ret == expected_wrapped) { \ wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ sizeof(t2)); \ } \ return (t1)expected_wrapped; \ } \ DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ static inline t1 name##_shared_unchecked( \ wasm_rt_shared_memory_t* mem, u64 addr, t1 expected, t1 replacement) { \ ATOMIC_ALIGNMENT_CHECK(addr, t2); \ t2 expected_wrapped = (t2)expected; \ t2 replacement_wrapped = (t2)replacement; \ atomic_compare_exchange_strong( \ (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ &expected_wrapped, replacement_wrapped); \ return (t1)expected_wrapped; \ } \ DEF_MEM_CHECKS2(name##_shared, _shared_, t2, return, t1, t1, t1) DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8); DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw16_cmpxchg_u, u32, u16); DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw_cmpxchg, u32, u32); DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw8_cmpxchg_u, u64, u8); DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw16_cmpxchg_u, u64, u16); DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw32_cmpxchg_u, u64, u32); DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw_cmpxchg, u64, u64); #define atomic_fence() atomic_thread_fence(memory_order_seq_cst)