wasm2c: atomic and shared mem operations using c11

4 files changed, 466 insertions, 433 deletions

diff --git a/src/binary-reader-ir.cc b/src/binary-reader-ir.cc
index 07546996..a918c39d 100644
--- a/src/binary-reader-ir.cc
+++ b/src/binary-reader-ir.cc
@@ -616,6 +616,9 @@ Result BinaryReaderIR::OnImportMemory(Index import_index,
   import->module_name = module_name;
   import->field_name = field_name;
   import->memory.page_limits = *page_limits;
+  if (import->memory.page_limits.is_shared) {
+    module_->features_used.threads = true;
+  }
   module_->AppendField(
       std::make_unique<ImportModuleField>(std::move(import), GetLocation()));
   return Result::Ok;
@@ -697,6 +700,9 @@ Result BinaryReaderIR::OnMemory(Index index, const Limits* page_limits) {
   auto field = std::make_unique<MemoryModuleField>(GetLocation());
   Memory& memory = field->memory;
   memory.page_limits = *page_limits;
+  if (memory.page_limits.is_shared) {
+    module_->features_used.threads = true;
+  }
   module_->AppendField(std::move(field));
   return Result::Ok;
 }
diff --git a/src/c-writer.cc b/src/c-writer.cc
index 0326c886..24bc0057 100644
--- a/src/c-writer.cc
+++ b/src/c-writer.cc
@@ -388,8 +388,8 @@ class CWriter {
   void WriteGlobal(const Global&, const std::string&);
   void WriteGlobalPtr(const Global&, const std::string&);
   void WriteMemories();
-  void WriteMemory(const std::string&);
-  void WriteMemoryPtr(const std::string&);
+  void WriteMemory(const std::string&, const Memory& memory);
+  void WriteMemoryPtr(const std::string&, const Memory& memory);
   void WriteTables();
   void WriteTable(const std::string&, const wabt::Type&);
   void WriteTablePtr(const std::string&, const Table&);
@@ -1328,6 +1328,15 @@ void CWriter::WriteGetFuncTypeDecl() {
         Newline());
 }
 
+static std::string GetMemoryTypeString(const Memory& memory) {
+  return memory.page_limits.is_shared ? "wasm_rt_shared_memory_t"
+                                      : "wasm_rt_memory_t";
+}
+
+static std::string GetMemoryAPIString(const Memory& memory, std::string api) {
+  return memory.page_limits.is_shared ? (api + "_shared") : api;
+}
+
 void CWriter::WriteInitExpr(const ExprList& expr_list) {
   if (expr_list.empty()) {
     WABT_UNREACHABLE;
@@ -1736,7 +1745,7 @@ void CWriter::BeginInstance() {
       }
 
       case ExternalKind::Memory: {
-        Write("wasm_rt_memory_t");
+        Write(GetMemoryTypeString(cast<MemoryImport>(import)->memory));
         break;
       }
 
@@ -1779,7 +1788,8 @@ void CWriter::BeginInstance() {
 
       case ExternalKind::Memory:
         WriteMemory(std::string("*") +
-                    ExportName(import->module_name, import->field_name));
+                        ExportName(import->module_name, import->field_name),
+                    cast<MemoryImport>(import)->memory);
         break;
 
       case ExternalKind::Table: {
@@ -2027,19 +2037,20 @@ void CWriter::WriteMemories() {
     bool is_import = memory_index < module_->num_memory_imports;
     if (!is_import) {
       WriteMemory(
-          DefineInstanceMemberName(ModuleFieldType::Memory, memory->name));
+          DefineInstanceMemberName(ModuleFieldType::Memory, memory->name),
+          *memory);
       Write(Newline());
     }
     ++memory_index;
   }
 }
 
-void CWriter::WriteMemory(const std::string& name) {
-  Write("wasm_rt_memory_t ", name, ";");
+void CWriter::WriteMemory(const std::string& name, const Memory& memory) {
+  Write(GetMemoryTypeString(memory), " ", name, ";");
 }
 
-void CWriter::WriteMemoryPtr(const std::string& name) {
-  Write("wasm_rt_memory_t* ", name, "(", ModuleInstanceTypeName(),
+void CWriter::WriteMemoryPtr(const std::string& name, const Memory& memory) {
+  Write(GetMemoryTypeString(memory), "* ", name, "(", ModuleInstanceTypeName(),
         "* instance)");
 }
 
@@ -2169,7 +2180,8 @@ void CWriter::WriteDataInitializers() {
         max = memory->page_limits.is_64 ? (static_cast<uint64_t>(1) << 48)
                                         : 65536;
       }
-      Write("wasm_rt_allocate_memory(",
+      std::string func = GetMemoryAPIString(*memory, "wasm_rt_allocate_memory");
+      Write(func, "(",
             ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", ",
             memory->page_limits.initial, ", ", max, ", ",
             memory->page_limits.is_64, ");", Newline());
@@ -2444,7 +2456,7 @@ void CWriter::WriteExports(CWriterPhase kind) {
       case ExternalKind::Memory: {
         const Memory* memory = module_->GetMemory(export_->var);
         internal_name = memory->name;
-        WriteMemoryPtr(mangled_name);
+        WriteMemoryPtr(mangled_name, *memory);
         break;
       }
 
@@ -2754,7 +2766,8 @@ void CWriter::WriteFree() {
     for (const Memory* memory : module_->memories) {
       bool is_import = memory_index < module_->num_memory_imports;
       if (!is_import) {
-        Write("wasm_rt_free_memory(",
+        std::string func = GetMemoryAPIString(*memory, "wasm_rt_free_memory");
+        Write(func, "(",
               ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ");",
               Newline());
       }
@@ -3706,7 +3719,8 @@ void CWriter::Write(const ExprList& exprs) {
         Memory* memory = module_->memories[module_->GetMemoryIndex(
             cast<MemoryGrowExpr>(&expr)->memidx)];
 
-        Write(StackVar(0), " = wasm_rt_grow_memory(",
+        std::string func = GetMemoryAPIString(*memory, "wasm_rt_grow_memory");
+        Write(StackVar(0), " = ", func, "(",
               ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", ",
               StackVar(0), ");", Newline());
         break;
@@ -4923,7 +4937,7 @@ void CWriter::Write(const ConvertExpr& expr) {
 }
 
 void CWriter::Write(const LoadExpr& expr) {
-  const char* func = nullptr;
+  std::string func;
   // clang-format off
   switch (expr.opcode) {
     case Opcode::I32Load: func = "i32_load"; break;
@@ -4954,6 +4968,7 @@ void CWriter::Write(const LoadExpr& expr) {
   // clang-format on
 
   Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)];
+  func = GetMemoryAPIString(*memory, func);
 
   Type result_type = expr.opcode.GetResultType();
   Write(StackVar(0, result_type), " = ", func, "(",
@@ -4967,7 +4982,7 @@ void CWriter::Write(const LoadExpr& expr) {
 }
 
 void CWriter::Write(const StoreExpr& expr) {
-  const char* func = nullptr;
+  std::string func;
   // clang-format off
   switch (expr.opcode) {
     case Opcode::I32Store: func = "i32_store"; break;
@@ -4987,6 +5002,7 @@ void CWriter::Write(const StoreExpr& expr) {
   // clang-format on
 
   Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)];
+  func = GetMemoryAPIString(*memory, func);
 
   Write(func, "(", ExternalInstancePtr(ModuleFieldType::Memory, memory->name),
         ", (u64)(", StackVar(1), ")");
@@ -5560,7 +5576,7 @@ void CWriter::Write(const LoadZeroExpr& expr) {
 }
 
 void CWriter::Write(const AtomicLoadExpr& expr) {
-  const char* func = nullptr;
+  std::string func;
   // clang-format off
   switch (expr.opcode) {
     case Opcode::I32AtomicLoad: func = "i32_atomic_load"; break;
@@ -5577,6 +5593,7 @@ void CWriter::Write(const AtomicLoadExpr& expr) {
   // clang-format on
 
   Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)];
+  func = GetMemoryAPIString(*memory, func);
 
   Type result_type = expr.opcode.GetResultType();
   Write(StackVar(0, result_type), " = ", func, "(",
@@ -5590,7 +5607,7 @@ void CWriter::Write(const AtomicLoadExpr& expr) {
 }
 
 void CWriter::Write(const AtomicStoreExpr& expr) {
-  const char* func = nullptr;
+  std::string func;
   // clang-format off
   switch (expr.opcode) {
     case Opcode::I32AtomicStore: func = "i32_atomic_store"; break;
@@ -5607,6 +5624,7 @@ void CWriter::Write(const AtomicStoreExpr& expr) {
   // clang-format on
 
   Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)];
+  func = GetMemoryAPIString(*memory, func);
 
   Write(func, "(", ExternalInstancePtr(ModuleFieldType::Memory, memory->name),
         ", (u64)(", StackVar(1), ")");
@@ -5617,7 +5635,7 @@ void CWriter::Write(const AtomicStoreExpr& expr) {
 }
 
 void CWriter::Write(const AtomicRmwExpr& expr) {
-  const char* func = nullptr;
+  std::string func;
   // clang-format off
   switch (expr.opcode) {
     case Opcode::I32AtomicRmwAdd: func = "i32_atomic_rmw_add"; break;
@@ -5668,6 +5686,8 @@ void CWriter::Write(const AtomicRmwExpr& expr) {
   // clang-format on
 
   Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)];
+  func = GetMemoryAPIString(*memory, func);
+
   Type result_type = expr.opcode.GetResultType();
 
   Write(StackVar(1, result_type), " = ", func, "(",
@@ -5681,7 +5701,7 @@ void CWriter::Write(const AtomicRmwExpr& expr) {
 }
 
 void CWriter::Write(const AtomicRmwCmpxchgExpr& expr) {
-  const char* func = nullptr;
+  std::string func;
   // clang-format off
   switch(expr.opcode) {
     case Opcode::I32AtomicRmwCmpxchg: func = "i32_atomic_rmw_cmpxchg"; break;
@@ -5697,6 +5717,8 @@ void CWriter::Write(const AtomicRmwCmpxchgExpr& expr) {
   // clang-format on
 
   Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)];
+  func = GetMemoryAPIString(*memory, func);
+
   Type result_type = expr.opcode.GetResultType();
 
   Write(StackVar(2, result_type), " = ", func, "(",
diff --git a/src/prebuilt/wasm2c_atomicops_source_declarations.cc b/src/prebuilt/wasm2c_atomicops_source_declarations.cc
index 5e82c15b..68b687f3 100644
--- a/src/prebuilt/wasm2c_atomicops_source_declarations.cc
+++ b/src/prebuilt/wasm2c_atomicops_source_declarations.cc
@@ -1,424 +1,430 @@
-const char* s_atomicops_source_declarations = R"w2c_template(#if defined(_MSC_VER)
+const char* s_atomicops_source_declarations = R"w2c_template(#include <stdatomic.h>
 )w2c_template"
 R"w2c_template(
-#include <intrin.h>
+#if WABT_BIG_ENDIAN
 )w2c_template"
-R"w2c_template(
-// Use MSVC intrinsics
+R"w2c_template(#error "wasm2c atomics not supported on big endian"
 )w2c_template"
-R"w2c_template(
-// For loads and stores, its not clear if we can rely on register width loads
+R"w2c_template(#endif
 )w2c_template"
-R"w2c_template(// and stores to be atomic as reported here
+R"w2c_template(
+#ifndef WASM_RT_C11_AVAILABLE
 )w2c_template"
-R"w2c_template(// https://learn.microsoft.com/en-us/windows/win32/sync/interlocked-variable-access?redirectedfrom=MSDN
+R"w2c_template(#error "C11 is required for Wasm threads and shared memory support"
 )w2c_template"
-R"w2c_template(// or if we have to reuse other instrinsics
+R"w2c_template(#endif
 )w2c_template"
-R"w2c_template(// https://stackoverflow.com/questions/42660091/atomic-load-in-c-with-msvc
+R"w2c_template(
+#define ATOMIC_ALIGNMENT_CHECK(addr, t1) \
 )w2c_template"
-R"w2c_template(// We reuse other intrinsics to be cautious
+R"w2c_template(  if (UNLIKELY(addr % sizeof(t1))) {     \
 )w2c_template"
-R"w2c_template(#define atomic_load_u8(a) _InterlockedOr8(a, 0)
+R"w2c_template(    TRAP(UNALIGNED);                     \
 )w2c_template"
-R"w2c_template(#define atomic_load_u16(a) _InterlockedOr16(a, 0)
+R"w2c_template(  }
 )w2c_template"
-R"w2c_template(#define atomic_load_u32(a) _InterlockedOr(a, 0)
+R"w2c_template(
+#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read)          \
 )w2c_template"
-R"w2c_template(#define atomic_load_u64(a) _InterlockedOr64(a, 0)
+R"w2c_template(  static inline t3 name(wasm_rt_shared_memory_t* mem, u64 addr) { \
 )w2c_template"
-R"w2c_template(
-#define atomic_store_u8(a, v) _InterlockedExchange8(a, v)
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                      \
 )w2c_template"
-R"w2c_template(#define atomic_store_u16(a, v) _InterlockedExchange16(a, v)
+R"w2c_template(    t1 result;                                                    \
 )w2c_template"
-R"w2c_template(#define atomic_store_u32(a, v) _InterlockedExchange(a, v)
+R"w2c_template(    result = atomic_load_explicit(                                \
 )w2c_template"
-R"w2c_template(#define atomic_store_u64(a, v) _InterlockedExchange64(a, v)
+R"w2c_template(        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)),    \
 )w2c_template"
-R"w2c_template(
-#define atomic_add_u8(a, v) _InterlockedExchangeAdd8(a, v)
+R"w2c_template(        memory_order_relaxed);                                    \
 )w2c_template"
-R"w2c_template(#define atomic_add_u16(a, v) _InterlockedExchangeAdd16(a, v)
+R"w2c_template(    force_read(result);                                           \
 )w2c_template"
-R"w2c_template(#define atomic_add_u32(a, v) _InterlockedExchangeAdd(a, v)
+R"w2c_template(    return (t3)(t2)result;                                        \
 )w2c_template"
-R"w2c_template(#define atomic_add_u64(a, v) _InterlockedExchangeAdd64(a, v)
+R"w2c_template(  }
 )w2c_template"
 R"w2c_template(
-#define atomic_sub_u8(a, v) _InterlockedExchangeAdd8(a, -(v))
+DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_sub_u16(a, v) _InterlockedExchangeAdd16(a, -(v))
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load_shared, u64, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_sub_u32(a, v) _InterlockedExchangeAdd(a, -(v))
+R"w2c_template(DEFINE_SHARED_LOAD(f32_load_shared, f32, f32, f32, FORCE_READ_FLOAT)
 )w2c_template"
-R"w2c_template(#define atomic_sub_u64(a, v) _InterlockedExchangeAdd64(a, -(v))
+R"w2c_template(DEFINE_SHARED_LOAD(f64_load_shared, f64, f64, f64, FORCE_READ_FLOAT)
 )w2c_template"
-R"w2c_template(
-#define atomic_and_u8(a, v) _InterlockedAnd8(a, v)
+R"w2c_template(DEFINE_SHARED_LOAD(i32_load8_s_shared, s8, s32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_and_u16(a, v) _InterlockedAnd16(a, v)
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load8_s_shared, s8, s64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_and_u32(a, v) _InterlockedAnd(a, v)
+R"w2c_template(DEFINE_SHARED_LOAD(i32_load8_u_shared, u8, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_and_u64(a, v) _InterlockedAnd64(a, v)
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load8_u_shared, u8, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(
-#define atomic_or_u8(a, v) _InterlockedOr8(a, v)
+R"w2c_template(DEFINE_SHARED_LOAD(i32_load16_s_shared, s16, s32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load16_s_shared, s16, s64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_or_u16(a, v) _InterlockedOr16(a, v)
+R"w2c_template(DEFINE_SHARED_LOAD(i32_load16_u_shared, u16, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_or_u32(a, v) _InterlockedOr(a, v)
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load16_u_shared, u16, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_or_u64(a, v) _InterlockedOr64(a, v)
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load32_s_shared, s32, s64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ_INT)
 )w2c_template"
 R"w2c_template(
-#define atomic_xor_u8(a, v) _InterlockedXor8(a, v)
+#define DEFINE_SHARED_STORE(name, t1, t2)                                     \
 )w2c_template"
-R"w2c_template(#define atomic_xor_u16(a, v) _InterlockedXor16(a, v)
+R"w2c_template(  static inline void name(wasm_rt_shared_memory_t* mem, u64 addr, t2 value) { \
 )w2c_template"
-R"w2c_template(#define atomic_xor_u32(a, v) _InterlockedXor(a, v)
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                                  \
 )w2c_template"
-R"w2c_template(#define atomic_xor_u64(a, v) _InterlockedXor64(a, v)
+R"w2c_template(    t1 wrapped = (t1)value;                                                   \
 )w2c_template"
-R"w2c_template(
-#define atomic_exchange_u8(a, v) _InterlockedExchange8(a, v)
+R"w2c_template(    atomic_store_explicit(                                                    \
 )w2c_template"
-R"w2c_template(#define atomic_exchange_u16(a, v) _InterlockedExchange16(a, v)
+R"w2c_template(        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped,       \
 )w2c_template"
-R"w2c_template(#define atomic_exchange_u32(a, v) _InterlockedExchange(a, v)
+R"w2c_template(        memory_order_relaxed);                                                \
 )w2c_template"
-R"w2c_template(#define atomic_exchange_u64(a, v) _InterlockedExchange64(a, v)
+R"w2c_template(  }
 )w2c_template"
 R"w2c_template(
-// clang-format off
+DEFINE_SHARED_STORE(i32_store_shared, u32, u32)
 )w2c_template"
-R"w2c_template(#define atomic_compare_exchange_u8(a, expected_ptr, desired) _InterlockedCompareExchange8(a, desired, *(expected_ptr))
+R"w2c_template(DEFINE_SHARED_STORE(i64_store_shared, u64, u64)
 )w2c_template"
-R"w2c_template(#define atomic_compare_exchange_u16(a, expected_ptr, desired) _InterlockedCompareExchange16(a, desired, *(expected_ptr))
+R"w2c_template(DEFINE_SHARED_STORE(f32_store_shared, f32, f32)
 )w2c_template"
-R"w2c_template(#define atomic_compare_exchange_u32(a, expected_ptr, desired) _InterlockedCompareExchange(a, desired, *(expected_ptr))
+R"w2c_template(DEFINE_SHARED_STORE(f64_store_shared, f64, f64)
 )w2c_template"
-R"w2c_template(#define atomic_compare_exchange_u64(a, expected_ptr, desired) _InterlockedCompareExchange64(a, desired, *(expected_ptr))
+R"w2c_template(DEFINE_SHARED_STORE(i32_store8_shared, u8, u32)
 )w2c_template"
-R"w2c_template(// clang-format on
+R"w2c_template(DEFINE_SHARED_STORE(i32_store16_shared, u16, u32)
 )w2c_template"
-R"w2c_template(
-#define atomic_fence() _ReadWriteBarrier()
+R"w2c_template(DEFINE_SHARED_STORE(i64_store8_shared, u8, u64)
 )w2c_template"
-R"w2c_template(
-#else
+R"w2c_template(DEFINE_SHARED_STORE(i64_store16_shared, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(i64_store32_shared, u32, u64)
 )w2c_template"
 R"w2c_template(
-// Use gcc/clang/icc intrinsics
+#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read)                    \
 )w2c_template"
-R"w2c_template(#define atomic_load_u8(a) __atomic_load_n((u8*)(a), __ATOMIC_SEQ_CST)
+R"w2c_template(  static inline t3 name(wasm_rt_memory_t* mem, u64 addr) {                  \
 )w2c_template"
-R"w2c_template(#define atomic_load_u16(a) __atomic_load_n((u16*)(a), __ATOMIC_SEQ_CST)
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                                \
 )w2c_template"
-R"w2c_template(#define atomic_load_u32(a) __atomic_load_n((u32*)(a), __ATOMIC_SEQ_CST)
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                       \
 )w2c_template"
-R"w2c_template(#define atomic_load_u64(a) __atomic_load_n((u64*)(a), __ATOMIC_SEQ_CST)
+R"w2c_template(    t1 result;                                                              \
 )w2c_template"
-R"w2c_template(
-#define atomic_store_u8(a, v) __atomic_store_n((u8*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1));   \
 )w2c_template"
-R"w2c_template(#define atomic_store_u16(a, v) __atomic_store_n((u16*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    force_read(result);                                                     \
 )w2c_template"
-R"w2c_template(#define atomic_store_u32(a, v) __atomic_store_n((u32*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    return (t3)(t2)result;                                                  \
 )w2c_template"
-R"w2c_template(#define atomic_store_u64(a, v) __atomic_store_n((u64*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(  }                                                                         \
 )w2c_template"
-R"w2c_template(
-#define atomic_add_u8(a, v) __atomic_fetch_add((u8*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(  static inline t3 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr) {  \
 )w2c_template"
-R"w2c_template(#define atomic_add_u16(a, v) __atomic_fetch_add((u16*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                                \
 )w2c_template"
-R"w2c_template(#define atomic_add_u32(a, v) __atomic_fetch_add((u32*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                       \
 )w2c_template"
-R"w2c_template(#define atomic_add_u64(a, v) __atomic_fetch_add((u64*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    t1 result;                                                              \
 )w2c_template"
-R"w2c_template(
-#define atomic_sub_u8(a, v) __atomic_fetch_sub((u8*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    result =                                                                \
 )w2c_template"
-R"w2c_template(#define atomic_sub_u16(a, v) __atomic_fetch_sub((u16*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(        atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \
 )w2c_template"
-R"w2c_template(#define atomic_sub_u32(a, v) __atomic_fetch_sub((u32*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    force_read(result);                                                     \
 )w2c_template"
-R"w2c_template(#define atomic_sub_u64(a, v) __atomic_fetch_sub((u64*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    return (t3)(t2)result;                                                  \
 )w2c_template"
-R"w2c_template(
-#define atomic_and_u8(a, v) __atomic_fetch_and((u8*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(  }
 )w2c_template"
-R"w2c_template(#define atomic_and_u16(a, v) __atomic_fetch_and((u16*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(
+DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_and_u32(a, v) __atomic_fetch_and((u32*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load, u64, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_and_u64(a, v) __atomic_fetch_and((u64*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(DEFINE_ATOMIC_LOAD(i32_atomic_load8_u, u8, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(
-#define atomic_or_u8(a, v) __atomic_fetch_or((u8*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load8_u, u8, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_or_u16(a, v) __atomic_fetch_or((u16*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(DEFINE_ATOMIC_LOAD(i32_atomic_load16_u, u16, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_or_u32(a, v) __atomic_fetch_or((u32*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(#define atomic_or_u64(a, v) __atomic_fetch_or((u64*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT)
 )w2c_template"
 R"w2c_template(
-#define atomic_xor_u8(a, v) __atomic_fetch_xor((u8*)(a), v, __ATOMIC_SEQ_CST)
+#define DEFINE_ATOMIC_STORE(name, t1, t2)                                  \
 )w2c_template"
-R"w2c_template(#define atomic_xor_u16(a, v) __atomic_fetch_xor((u16*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(  static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) {     \
 )w2c_template"
-R"w2c_template(#define atomic_xor_u32(a, v) __atomic_fetch_xor((u32*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                               \
 )w2c_template"
-R"w2c_template(#define atomic_xor_u64(a, v) __atomic_fetch_xor((u64*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
 )w2c_template"
-R"w2c_template(
-// clang-format off
+R"w2c_template(    t1 wrapped = (t1)value;                                                \
 )w2c_template"
-R"w2c_template(#define atomic_exchange_u8(a, v) __atomic_exchange_n((u8*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(    wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \
 )w2c_template"
-R"w2c_template(#define atomic_exchange_u16(a, v) __atomic_exchange_n((u16*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(  }                                                                        \
 )w2c_template"
-R"w2c_template(#define atomic_exchange_u32(a, v) __atomic_exchange_n((u32*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(  static inline void name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
 )w2c_template"
-R"w2c_template(#define atomic_exchange_u64(a, v) __atomic_exchange_n((u64*)(a), v, __ATOMIC_SEQ_CST)
+R"w2c_template(                                   t2 value) {                             \
 )w2c_template"
-R"w2c_template(// clang-format on
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                               \
 )w2c_template"
-R"w2c_template(
-#define __atomic_compare_exchange_helper(a, expected_ptr, desired)        \
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
 )w2c_template"
-R"w2c_template(  (__atomic_compare_exchange_n(a, expected_ptr, desired, 0 /* is_weak */, \
+R"w2c_template(    t1 wrapped = (t1)value;                                                \
+)w2c_template"
+R"w2c_template(    atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)),    \
 )w2c_template"
-R"w2c_template(                               __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST),       \
+R"w2c_template(                 wrapped);                                                 \
 )w2c_template"
-R"w2c_template(   *(expected_ptr))
+R"w2c_template(  }
 )w2c_template"
 R"w2c_template(
-// clang-format off
+DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store, u64, u64)
 )w2c_template"
-R"w2c_template(#define atomic_compare_exchange_u8(a, expected_ptr, desired) __atomic_compare_exchange_helper((u8*)(a), expected_ptr, desired)
+R"w2c_template(DEFINE_ATOMIC_STORE(i32_atomic_store8, u8, u32)
 )w2c_template"
-R"w2c_template(#define atomic_compare_exchange_u16(a, expected_ptr, desired) __atomic_compare_exchange_helper((u16*)(a), expected_ptr, desired)
+R"w2c_template(DEFINE_ATOMIC_STORE(i32_atomic_store16, u16, u32)
 )w2c_template"
-R"w2c_template(#define atomic_compare_exchange_u32(a, expected_ptr, desired) __atomic_compare_exchange_helper((u32*)(a), expected_ptr, desired)
+R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store8, u8, u64)
 )w2c_template"
-R"w2c_template(#define atomic_compare_exchange_u64(a, expected_ptr, desired) __atomic_compare_exchange_helper((u64*)(a), expected_ptr, desired)
+R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64)
 )w2c_template"
-R"w2c_template(// clang-format on
+R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64)
 )w2c_template"
 R"w2c_template(
-#define atomic_fence() __atomic_thread_fence(__ATOMIC_SEQ_CST)
+#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2)                      \
 )w2c_template"
-R"w2c_template(
-#endif
+R"w2c_template(  static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) {     \
 )w2c_template"
-R"w2c_template(
-#define ATOMIC_ALIGNMENT_CHECK(addr, t1) \
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                             \
 )w2c_template"
-R"w2c_template(  if (UNLIKELY(addr % sizeof(t1))) {     \
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                    \
 )w2c_template"
-R"w2c_template(    TRAP(UNALIGNED);                     \
+R"w2c_template(    t1 wrapped = (t1)value;                                              \
 )w2c_template"
-R"w2c_template(  }
+R"w2c_template(    t1 ret;                                                              \
 )w2c_template"
-R"w2c_template(
-#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read)                  \
+R"w2c_template(    wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1));   \
+)w2c_template"
+R"w2c_template(    ret = ret op wrapped;                                                \
+)w2c_template"
+R"w2c_template(    wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &ret, sizeof(t1));   \
 )w2c_template"
-R"w2c_template(  static inline t3 name(wasm_rt_memory_t* mem, u64 addr) {                \
+R"w2c_template(    return (t2)ret;                                                      \
 )w2c_template"
-R"w2c_template(    MEMCHECK(mem, addr, t1);                                              \
+R"w2c_template(  }                                                                      \
 )w2c_template"
-R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                     \
+R"w2c_template(  static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
 )w2c_template"
-R"w2c_template(    t1 result;                                                            \
+R"w2c_template(                                 t2 value) {                             \
 )w2c_template"
-R"w2c_template(    wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                             \
 )w2c_template"
-R"w2c_template(    result = atomic_load_##t1(MEM_ADDR(mem, addr, sizeof(t1)));           \
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                    \
 )w2c_template"
-R"w2c_template(    force_read(result);                                                   \
+R"w2c_template(    t1 wrapped = (t1)value;                                              \
 )w2c_template"
-R"w2c_template(    return (t3)(t2)result;                                                \
+R"w2c_template(    t1 ret = atomic_##opname(                                            \
+)w2c_template"
+R"w2c_template(        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \
+)w2c_template"
+R"w2c_template(    return (t2)ret;                                                      \
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
 R"w2c_template(
-DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load, u64, u64, u64, FORCE_READ_INT)
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_add_u, fetch_add, +, u16, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_LOAD(i32_atomic_load8_u, u8, u32, u32, FORCE_READ_INT)
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_add, fetch_add, +, u32, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load8_u, u8, u64, u64, FORCE_READ_INT)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_add_u, fetch_add, +, u8, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_LOAD(i32_atomic_load16_u, u16, u32, u32, FORCE_READ_INT)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_add_u, fetch_add, +, u16, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ_INT)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_add_u, fetch_add, +, u32, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_add, fetch_add, +, u64, u64)
 )w2c_template"
 R"w2c_template(
-#define DEFINE_ATOMIC_STORE(name, t1, t2)                              \
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_sub_u, fetch_sub, -, u8, u32)
 )w2c_template"
-R"w2c_template(  static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_sub_u, fetch_sub, -, u16, u32)
 )w2c_template"
-R"w2c_template(    MEMCHECK(mem, addr, t1);                                           \
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_sub, fetch_sub, -, u32, u32)
 )w2c_template"
-R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                  \
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_sub_u, fetch_sub, -, u8, u64)
 )w2c_template"
-R"w2c_template(    t1 wrapped = (t1)value;                                            \
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_sub_u, fetch_sub, -, u16, u64)
 )w2c_template"
-R"w2c_template(    atomic_store_##t1(MEM_ADDR(mem, addr, sizeof(t1)), wrapped);       \
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_sub_u, fetch_sub, -, u32, u64)
 )w2c_template"
-R"w2c_template(  }
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_sub, fetch_sub, -, u64, u64)
 )w2c_template"
 R"w2c_template(
-DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_and_u, fetch_and, &, u8, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store, u64, u64)
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_and_u, fetch_and, &, u16, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_STORE(i32_atomic_store8, u8, u32)
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_and, fetch_and, &, u32, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_STORE(i32_atomic_store16, u16, u32)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_and_u, fetch_and, &, u8, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store8, u8, u64)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_and_u, fetch_and, &, u16, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_and_u, fetch_and, &, u32, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_and, fetch_and, &, u64, u64)
 )w2c_template"
 R"w2c_template(
-#define DEFINE_ATOMIC_RMW(name, op, t1, t2)                                \
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_or_u, fetch_or, |, u8, u32)
 )w2c_template"
-R"w2c_template(  static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) {       \
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_or_u, fetch_or, |, u16, u32)
 )w2c_template"
-R"w2c_template(    MEMCHECK(mem, addr, t1);                                               \
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_or, fetch_or, |, u32, u32)
 )w2c_template"
-R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_or_u, fetch_or, |, u8, u64)
 )w2c_template"
-R"w2c_template(    t1 wrapped = (t1)value;                                                \
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_or_u, fetch_or, |, u16, u64)
 )w2c_template"
-R"w2c_template(    t1 ret = atomic_##op##_##t1(MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_or_u, fetch_or, |, u32, u64)
 )w2c_template"
-R"w2c_template(    return (t2)ret;                                                        \
-)w2c_template"
-R"w2c_template(  }
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_or, fetch_or, |, u64, u64)
 )w2c_template"
 R"w2c_template(
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, add, u8, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_xor_u, fetch_xor, ^, u8, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_add_u, add, u16, u32)
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_xor_u, fetch_xor, ^, u16, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_add, add, u32, u32)
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_xor, fetch_xor, ^, u32, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_add_u, add, u8, u64)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_xor_u, fetch_xor, ^, u8, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_add_u, add, u16, u64)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xor_u, fetch_xor, ^, u16, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_add_u, add, u32, u64)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_add, add, u64, u64)
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64)
 )w2c_template"
 R"w2c_template(
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_sub_u, sub, u8, u32)
+#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2)                               \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_sub_u, sub, u16, u32)
+R"w2c_template(  static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) {           \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_sub, sub, u32, u32)
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                                   \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_sub_u, sub, u8, u64)
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                          \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_sub_u, sub, u16, u64)
+R"w2c_template(    t1 wrapped = (t1)value;                                                    \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_sub_u, sub, u32, u64)
+R"w2c_template(    t1 ret;                                                                    \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_sub, sub, u64, u64)
+R"w2c_template(    wasm_rt_memcpy(&ret, &mem->data[addr], sizeof(t1));                        \
 )w2c_template"
-R"w2c_template(
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_and_u, and, u8, u32)
+R"w2c_template(    wasm_rt_memcpy(&mem->data[addr], &wrapped, sizeof(t1));                    \
+)w2c_template"
+R"w2c_template(    return (t2)ret;                                                            \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_and_u, and, u16, u32)
+R"w2c_template(  }                                                                            \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_and, and, u32, u32)
+R"w2c_template(  static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr,       \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_and_u, and, u8, u64)
+R"w2c_template(                                 t2 value) {                                   \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_and_u, and, u16, u64)
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                                   \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_and_u, and, u32, u64)
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                          \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_and, and, u64, u64)
+R"w2c_template(    t1 wrapped = (t1)value;                                                    \
+)w2c_template"
+R"w2c_template(    t1 ret = atomic_##opname((_Atomic volatile t1*)&mem->data[addr], wrapped); \
+)w2c_template"
+R"w2c_template(    return (t2)ret;                                                            \
+)w2c_template"
+R"w2c_template(  }
 )w2c_template"
 R"w2c_template(
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_or_u, or, u8, u32)
+DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_or_u, or, u16, u32)
+R"w2c_template(DEFINE_ATOMIC_XCHG(i32_atomic_rmw16_xchg_u, exchange, u16, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_or, or, u32, u32)
+R"w2c_template(DEFINE_ATOMIC_XCHG(i32_atomic_rmw_xchg, exchange, u32, u32)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_or_u, or, u8, u64)
+R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw8_xchg_u, exchange, u8, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_or_u, or, u16, u64)
+R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw16_xchg_u, exchange, u16, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_or_u, or, u32, u64)
+R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64)
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_or, or, u64, u64)
+R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64)
 )w2c_template"
 R"w2c_template(
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_xor_u, xor, u8, u32)
+#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2)                                \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_xor_u, xor, u16, u32)
+R"w2c_template(  static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected,       \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_xor, xor, u32, u32)
+R"w2c_template(                        t1 replacement) {                                   \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_xor_u, xor, u8, u64)
+R"w2c_template(    MEMCHECK(mem, addr, t2);                                                \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xor_u, xor, u16, u64)
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t2);                                       \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, xor, u32, u64)
+R"w2c_template(    t2 expected_wrapped = (t2)expected;                                     \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, xor, u64, u64)
+R"w2c_template(    t2 replacement_wrapped = (t2)replacement;                               \
 )w2c_template"
-R"w2c_template(
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_xchg_u, exchange, u8, u32)
+R"w2c_template(    t2 ret;                                                                 \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_xchg_u, exchange, u16, u32)
+R"w2c_template(    wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2));      \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_xchg, exchange, u32, u32)
+R"w2c_template(    if (ret == expected_wrapped) {                                          \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_xchg_u, exchange, u8, u64)
+R"w2c_template(      wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xchg_u, exchange, u16, u64)
+R"w2c_template(                     sizeof(t2));                                           \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xchg_u, exchange, u32, u64)
+R"w2c_template(    }                                                                       \
 )w2c_template"
-R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xchg, exchange, u64, u64)
+R"w2c_template(    return (t1)expected_wrapped;                                            \
 )w2c_template"
-R"w2c_template(
-#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2)                                  \
+R"w2c_template(  }                                                                         \
 )w2c_template"
-R"w2c_template(  static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected,         \
+R"w2c_template(  static inline t1 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr,    \
 )w2c_template"
-R"w2c_template(                        t1 replacement) {                                     \
+R"w2c_template(                                 t1 expected, t1 replacement) {             \
 )w2c_template"
-R"w2c_template(    MEMCHECK(mem, addr, t2);                                                  \
+R"w2c_template(    MEMCHECK(mem, addr, t2);                                                \
 )w2c_template"
-R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t2);                                         \
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t2);                                       \
 )w2c_template"
-R"w2c_template(    t2 expected_wrapped = (t2)expected;                                       \
+R"w2c_template(    t2 expected_wrapped = (t2)expected;                                     \
 )w2c_template"
-R"w2c_template(    t2 replacement_wrapped = (t2)replacement;                                 \
+R"w2c_template(    t2 replacement_wrapped = (t2)replacement;                               \
 )w2c_template"
-R"w2c_template(    t2 old =                                                                  \
+R"w2c_template(    atomic_compare_exchange_strong(                                         \
 )w2c_template"
-R"w2c_template(        atomic_compare_exchange_##t2(MEM_ADDR(mem, addr, sizeof(t2)),         \
+R"w2c_template(        (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)),              \
 )w2c_template"
-R"w2c_template(                                     &expected_wrapped, replacement_wrapped); \
+R"w2c_template(        &expected_wrapped, replacement_wrapped);                            \
 )w2c_template"
-R"w2c_template(    return (t1)old;                                                           \
+R"w2c_template(    return (t1)expected_wrapped;                                            \
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
@@ -437,4 +443,7 @@ R"w2c_template(DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw32_cmpxchg_u, u64, u32);
 )w2c_template"
 R"w2c_template(DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw_cmpxchg, u64, u64);
 )w2c_template"
+R"w2c_template(
+#define atomic_fence() atomic_thread_fence(memory_order_seq_cst)
+)w2c_template"
 ;
diff --git a/src/template/wasm2c_atomicops.declarations.c b/src/template/wasm2c_atomicops.declarations.c
index 976f7f95..5d319991 100644
--- a/src/template/wasm2c_atomicops.declarations.c
+++ b/src/template/wasm2c_atomicops.declarations.c
@@ -1,123 +1,11 @@
-#if defined(_MSC_VER)
-
-#include <intrin.h>
-
-// Use MSVC intrinsics
-
-// For loads and stores, its not clear if we can rely on register width loads
-// and stores to be atomic as reported here
-// https://learn.microsoft.com/en-us/windows/win32/sync/interlocked-variable-access?redirectedfrom=MSDN
-// or if we have to reuse other instrinsics
-// https://stackoverflow.com/questions/42660091/atomic-load-in-c-with-msvc
-// We reuse other intrinsics to be cautious
-#define atomic_load_u8(a) _InterlockedOr8(a, 0)
-#define atomic_load_u16(a) _InterlockedOr16(a, 0)
-#define atomic_load_u32(a) _InterlockedOr(a, 0)
-#define atomic_load_u64(a) _InterlockedOr64(a, 0)
-
-#define atomic_store_u8(a, v) _InterlockedExchange8(a, v)
-#define atomic_store_u16(a, v) _InterlockedExchange16(a, v)
-#define atomic_store_u32(a, v) _InterlockedExchange(a, v)
-#define atomic_store_u64(a, v) _InterlockedExchange64(a, v)
-
-#define atomic_add_u8(a, v) _InterlockedExchangeAdd8(a, v)
-#define atomic_add_u16(a, v) _InterlockedExchangeAdd16(a, v)
-#define atomic_add_u32(a, v) _InterlockedExchangeAdd(a, v)
-#define atomic_add_u64(a, v) _InterlockedExchangeAdd64(a, v)
-
-#define atomic_sub_u8(a, v) _InterlockedExchangeAdd8(a, -(v))
-#define atomic_sub_u16(a, v) _InterlockedExchangeAdd16(a, -(v))
-#define atomic_sub_u32(a, v) _InterlockedExchangeAdd(a, -(v))
-#define atomic_sub_u64(a, v) _InterlockedExchangeAdd64(a, -(v))
-
-#define atomic_and_u8(a, v) _InterlockedAnd8(a, v)
-#define atomic_and_u16(a, v) _InterlockedAnd16(a, v)
-#define atomic_and_u32(a, v) _InterlockedAnd(a, v)
-#define atomic_and_u64(a, v) _InterlockedAnd64(a, v)
-
-#define atomic_or_u8(a, v) _InterlockedOr8(a, v)
-#define atomic_or_u16(a, v) _InterlockedOr16(a, v)
-#define atomic_or_u32(a, v) _InterlockedOr(a, v)
-#define atomic_or_u64(a, v) _InterlockedOr64(a, v)
-
-#define atomic_xor_u8(a, v) _InterlockedXor8(a, v)
-#define atomic_xor_u16(a, v) _InterlockedXor16(a, v)
-#define atomic_xor_u32(a, v) _InterlockedXor(a, v)
-#define atomic_xor_u64(a, v) _InterlockedXor64(a, v)
-
-#define atomic_exchange_u8(a, v) _InterlockedExchange8(a, v)
-#define atomic_exchange_u16(a, v) _InterlockedExchange16(a, v)
-#define atomic_exchange_u32(a, v) _InterlockedExchange(a, v)
-#define atomic_exchange_u64(a, v) _InterlockedExchange64(a, v)
-
-// clang-format off
-#define atomic_compare_exchange_u8(a, expected_ptr, desired) _InterlockedCompareExchange8(a, desired, *(expected_ptr))
-#define atomic_compare_exchange_u16(a, expected_ptr, desired) _InterlockedCompareExchange16(a, desired, *(expected_ptr))
-#define atomic_compare_exchange_u32(a, expected_ptr, desired) _InterlockedCompareExchange(a, desired, *(expected_ptr))
-#define atomic_compare_exchange_u64(a, expected_ptr, desired) _InterlockedCompareExchange64(a, desired, *(expected_ptr))
-// clang-format on
-
-#define atomic_fence() _ReadWriteBarrier()
-
-#else
-
-// Use gcc/clang/icc intrinsics
-#define atomic_load_u8(a) __atomic_load_n((u8*)(a), __ATOMIC_SEQ_CST)
-#define atomic_load_u16(a) __atomic_load_n((u16*)(a), __ATOMIC_SEQ_CST)
-#define atomic_load_u32(a) __atomic_load_n((u32*)(a), __ATOMIC_SEQ_CST)
-#define atomic_load_u64(a) __atomic_load_n((u64*)(a), __ATOMIC_SEQ_CST)
-
-#define atomic_store_u8(a, v) __atomic_store_n((u8*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_store_u16(a, v) __atomic_store_n((u16*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_store_u32(a, v) __atomic_store_n((u32*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_store_u64(a, v) __atomic_store_n((u64*)(a), v, __ATOMIC_SEQ_CST)
-
-#define atomic_add_u8(a, v) __atomic_fetch_add((u8*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_add_u16(a, v) __atomic_fetch_add((u16*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_add_u32(a, v) __atomic_fetch_add((u32*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_add_u64(a, v) __atomic_fetch_add((u64*)(a), v, __ATOMIC_SEQ_CST)
-
-#define atomic_sub_u8(a, v) __atomic_fetch_sub((u8*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_sub_u16(a, v) __atomic_fetch_sub((u16*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_sub_u32(a, v) __atomic_fetch_sub((u32*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_sub_u64(a, v) __atomic_fetch_sub((u64*)(a), v, __ATOMIC_SEQ_CST)
-
-#define atomic_and_u8(a, v) __atomic_fetch_and((u8*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_and_u16(a, v) __atomic_fetch_and((u16*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_and_u32(a, v) __atomic_fetch_and((u32*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_and_u64(a, v) __atomic_fetch_and((u64*)(a), v, __ATOMIC_SEQ_CST)
-
-#define atomic_or_u8(a, v) __atomic_fetch_or((u8*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_or_u16(a, v) __atomic_fetch_or((u16*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_or_u32(a, v) __atomic_fetch_or((u32*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_or_u64(a, v) __atomic_fetch_or((u64*)(a), v, __ATOMIC_SEQ_CST)
-
-#define atomic_xor_u8(a, v) __atomic_fetch_xor((u8*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_xor_u16(a, v) __atomic_fetch_xor((u16*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_xor_u32(a, v) __atomic_fetch_xor((u32*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_xor_u64(a, v) __atomic_fetch_xor((u64*)(a), v, __ATOMIC_SEQ_CST)
-
-// clang-format off
-#define atomic_exchange_u8(a, v) __atomic_exchange_n((u8*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_exchange_u16(a, v) __atomic_exchange_n((u16*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_exchange_u32(a, v) __atomic_exchange_n((u32*)(a), v, __ATOMIC_SEQ_CST)
-#define atomic_exchange_u64(a, v) __atomic_exchange_n((u64*)(a), v, __ATOMIC_SEQ_CST)
-// clang-format on
-
-#define __atomic_compare_exchange_helper(a, expected_ptr, desired)        \
-  (__atomic_compare_exchange_n(a, expected_ptr, desired, 0 /* is_weak */, \
-                               __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST),       \
-   *(expected_ptr))
-
-// clang-format off
-#define atomic_compare_exchange_u8(a, expected_ptr, desired) __atomic_compare_exchange_helper((u8*)(a), expected_ptr, desired)
-#define atomic_compare_exchange_u16(a, expected_ptr, desired) __atomic_compare_exchange_helper((u16*)(a), expected_ptr, desired)
-#define atomic_compare_exchange_u32(a, expected_ptr, desired) __atomic_compare_exchange_helper((u32*)(a), expected_ptr, desired)
-#define atomic_compare_exchange_u64(a, expected_ptr, desired) __atomic_compare_exchange_helper((u64*)(a), expected_ptr, desired)
-// clang-format on
-
-#define atomic_fence() __atomic_thread_fence(__ATOMIC_SEQ_CST)
+#include <stdatomic.h>
 
+#if WABT_BIG_ENDIAN
+#error "wasm2c atomics not supported on big endian"
+#endif
+
+#ifndef WASM_RT_C11_AVAILABLE
+#error "C11 is required for Wasm threads and shared memory support"
 #endif
 
 #define ATOMIC_ALIGNMENT_CHECK(addr, t1) \
@@ -125,15 +13,68 @@
     TRAP(UNALIGNED);                     \
   }
 
-#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read)                  \
-  static inline t3 name(wasm_rt_memory_t* mem, u64 addr) {                \
-    MEMCHECK(mem, addr, t1);                                              \
-    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                     \
-    t1 result;                                                            \
-    wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \
-    result = atomic_load_##t1(MEM_ADDR(mem, addr, sizeof(t1)));           \
-    force_read(result);                                                   \
-    return (t3)(t2)result;                                                \
+#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read)          \
+  static inline t3 name(wasm_rt_shared_memory_t* mem, u64 addr) { \
+    MEMCHECK(mem, addr, t1);                                      \
+    t1 result;                                                    \
+    result = atomic_load_explicit(                                \
+        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)),    \
+        memory_order_relaxed);                                    \
+    force_read(result);                                           \
+    return (t3)(t2)result;                                        \
+  }
+
+DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i64_load_shared, u64, u64, u64, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(f32_load_shared, f32, f32, f32, FORCE_READ_FLOAT)
+DEFINE_SHARED_LOAD(f64_load_shared, f64, f64, f64, FORCE_READ_FLOAT)
+DEFINE_SHARED_LOAD(i32_load8_s_shared, s8, s32, u32, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i64_load8_s_shared, s8, s64, u64, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i32_load8_u_shared, u8, u32, u32, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i64_load8_u_shared, u8, u64, u64, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i32_load16_s_shared, s16, s32, u32, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i64_load16_s_shared, s16, s64, u64, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i32_load16_u_shared, u16, u32, u32, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i64_load16_u_shared, u16, u64, u64, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i64_load32_s_shared, s32, s64, u64, FORCE_READ_INT)
+DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ_INT)
+
+#define DEFINE_SHARED_STORE(name, t1, t2)                                     \
+  static inline void name(wasm_rt_shared_memory_t* mem, u64 addr, t2 value) { \
+    MEMCHECK(mem, addr, t1);                                                  \
+    t1 wrapped = (t1)value;                                                   \
+    atomic_store_explicit(                                                    \
+        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped,       \
+        memory_order_relaxed);                                                \
+  }
+
+DEFINE_SHARED_STORE(i32_store_shared, u32, u32)
+DEFINE_SHARED_STORE(i64_store_shared, u64, u64)
+DEFINE_SHARED_STORE(f32_store_shared, f32, f32)
+DEFINE_SHARED_STORE(f64_store_shared, f64, f64)
+DEFINE_SHARED_STORE(i32_store8_shared, u8, u32)
+DEFINE_SHARED_STORE(i32_store16_shared, u16, u32)
+DEFINE_SHARED_STORE(i64_store8_shared, u8, u64)
+DEFINE_SHARED_STORE(i64_store16_shared, u16, u64)
+DEFINE_SHARED_STORE(i64_store32_shared, u32, u64)
+
+#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read)                    \
+  static inline t3 name(wasm_rt_memory_t* mem, u64 addr) {                  \
+    MEMCHECK(mem, addr, t1);                                                \
+    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                       \
+    t1 result;                                                              \
+    wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1));   \
+    force_read(result);                                                     \
+    return (t3)(t2)result;                                                  \
+  }                                                                         \
+  static inline t3 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr) {  \
+    MEMCHECK(mem, addr, t1);                                                \
+    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                       \
+    t1 result;                                                              \
+    result =                                                                \
+        atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \
+    force_read(result);                                                     \
+    return (t3)(t2)result;                                                  \
   }
 
 DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT)
@@ -144,12 +85,20 @@ DEFINE_ATOMIC_LOAD(i32_atomic_load16_u, u16, u32, u32, FORCE_READ_INT)
 DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ_INT)
 DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT)
 
-#define DEFINE_ATOMIC_STORE(name, t1, t2)                              \
-  static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \
-    MEMCHECK(mem, addr, t1);                                           \
-    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                  \
-    t1 wrapped = (t1)value;                                            \
-    atomic_store_##t1(MEM_ADDR(mem, addr, sizeof(t1)), wrapped);       \
+#define DEFINE_ATOMIC_STORE(name, t1, t2)                                  \
+  static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) {     \
+    MEMCHECK(mem, addr, t1);                                               \
+    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
+    t1 wrapped = (t1)value;                                                \
+    wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \
+  }                                                                        \
+  static inline void name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
+                                   t2 value) {                             \
+    MEMCHECK(mem, addr, t1);                                               \
+    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
+    t1 wrapped = (t1)value;                                                \
+    atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)),    \
+                 wrapped);                                                 \
   }
 
 DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32)
@@ -160,74 +109,119 @@ DEFINE_ATOMIC_STORE(i64_atomic_store8, u8, u64)
 DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64)
 DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64)
 
-#define DEFINE_ATOMIC_RMW(name, op, t1, t2)                                \
-  static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) {       \
-    MEMCHECK(mem, addr, t1);                                               \
-    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
-    t1 wrapped = (t1)value;                                                \
-    t1 ret = atomic_##op##_##t1(MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \
-    return (t2)ret;                                                        \
+#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2)                      \
+  static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) {     \
+    MEMCHECK(mem, addr, t1);                                             \
+    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                    \
+    t1 wrapped = (t1)value;                                              \
+    t1 ret;                                                              \
+    wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1));   \
+    ret = ret op wrapped;                                                \
+    wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &ret, sizeof(t1));   \
+    return (t2)ret;                                                      \
+  }                                                                      \
+  static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
+                                 t2 value) {                             \
+    MEMCHECK(mem, addr, t1);                                             \
+    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                    \
+    t1 wrapped = (t1)value;                                              \
+    t1 ret = atomic_##opname(                                            \
+        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \
+    return (t2)ret;                                                      \
+  }
+
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw16_add_u, fetch_add, +, u16, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw_add, fetch_add, +, u32, u32)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw8_add_u, fetch_add, +, u8, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw16_add_u, fetch_add, +, u16, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw32_add_u, fetch_add, +, u32, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw_add, fetch_add, +, u64, u64)
+
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_sub_u, fetch_sub, -, u8, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw16_sub_u, fetch_sub, -, u16, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw_sub, fetch_sub, -, u32, u32)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw8_sub_u, fetch_sub, -, u8, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw16_sub_u, fetch_sub, -, u16, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw32_sub_u, fetch_sub, -, u32, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw_sub, fetch_sub, -, u64, u64)
+
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_and_u, fetch_and, &, u8, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw16_and_u, fetch_and, &, u16, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw_and, fetch_and, &, u32, u32)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw8_and_u, fetch_and, &, u8, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw16_and_u, fetch_and, &, u16, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw32_and_u, fetch_and, &, u32, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw_and, fetch_and, &, u64, u64)
+
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_or_u, fetch_or, |, u8, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw16_or_u, fetch_or, |, u16, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw_or, fetch_or, |, u32, u32)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw8_or_u, fetch_or, |, u8, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw16_or_u, fetch_or, |, u16, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw32_or_u, fetch_or, |, u32, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw_or, fetch_or, |, u64, u64)
+
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_xor_u, fetch_xor, ^, u8, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw16_xor_u, fetch_xor, ^, u16, u32)
+DEFINE_ATOMIC_RMW(i32_atomic_rmw_xor, fetch_xor, ^, u32, u32)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw8_xor_u, fetch_xor, ^, u8, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xor_u, fetch_xor, ^, u16, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64)
+DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64)
+
+#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2)                               \
+  static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) {           \
+    MEMCHECK(mem, addr, t1);                                                   \
+    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                          \
+    t1 wrapped = (t1)value;                                                    \
+    t1 ret;                                                                    \
+    wasm_rt_memcpy(&ret, &mem->data[addr], sizeof(t1));                        \
+    wasm_rt_memcpy(&mem->data[addr], &wrapped, sizeof(t1));                    \
+    return (t2)ret;                                                            \
+  }                                                                            \
+  static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr,       \
+                                 t2 value) {                                   \
+    MEMCHECK(mem, addr, t1);                                                   \
+    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                          \
+    t1 wrapped = (t1)value;                                                    \
+    t1 ret = atomic_##opname((_Atomic volatile t1*)&mem->data[addr], wrapped); \
+    return (t2)ret;                                                            \
   }
 
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, add, u8, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw16_add_u, add, u16, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw_add, add, u32, u32)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw8_add_u, add, u8, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw16_add_u, add, u16, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw32_add_u, add, u32, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw_add, add, u64, u64)
-
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_sub_u, sub, u8, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw16_sub_u, sub, u16, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw_sub, sub, u32, u32)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw8_sub_u, sub, u8, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw16_sub_u, sub, u16, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw32_sub_u, sub, u32, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw_sub, sub, u64, u64)
-
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_and_u, and, u8, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw16_and_u, and, u16, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw_and, and, u32, u32)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw8_and_u, and, u8, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw16_and_u, and, u16, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw32_and_u, and, u32, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw_and, and, u64, u64)
-
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_or_u, or, u8, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw16_or_u, or, u16, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw_or, or, u32, u32)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw8_or_u, or, u8, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw16_or_u, or, u16, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw32_or_u, or, u32, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw_or, or, u64, u64)
-
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_xor_u, xor, u8, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw16_xor_u, xor, u16, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw_xor, xor, u32, u32)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw8_xor_u, xor, u8, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xor_u, xor, u16, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, xor, u32, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, xor, u64, u64)
-
-DEFINE_ATOMIC_RMW(i32_atomic_rmw8_xchg_u, exchange, u8, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw16_xchg_u, exchange, u16, u32)
-DEFINE_ATOMIC_RMW(i32_atomic_rmw_xchg, exchange, u32, u32)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw8_xchg_u, exchange, u8, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xchg_u, exchange, u16, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xchg_u, exchange, u32, u64)
-DEFINE_ATOMIC_RMW(i64_atomic_rmw_xchg, exchange, u64, u64)
-
-#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2)                                  \
-  static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected,         \
-                        t1 replacement) {                                     \
-    MEMCHECK(mem, addr, t2);                                                  \
-    ATOMIC_ALIGNMENT_CHECK(addr, t2);                                         \
-    t2 expected_wrapped = (t2)expected;                                       \
-    t2 replacement_wrapped = (t2)replacement;                                 \
-    t2 old =                                                                  \
-        atomic_compare_exchange_##t2(MEM_ADDR(mem, addr, sizeof(t2)),         \
-                                     &expected_wrapped, replacement_wrapped); \
-    return (t1)old;                                                           \
+DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32)
+DEFINE_ATOMIC_XCHG(i32_atomic_rmw16_xchg_u, exchange, u16, u32)
+DEFINE_ATOMIC_XCHG(i32_atomic_rmw_xchg, exchange, u32, u32)
+DEFINE_ATOMIC_XCHG(i64_atomic_rmw8_xchg_u, exchange, u8, u64)
+DEFINE_ATOMIC_XCHG(i64_atomic_rmw16_xchg_u, exchange, u16, u64)
+DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64)
+DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64)
+
+#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2)                                \
+  static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected,       \
+                        t1 replacement) {                                   \
+    MEMCHECK(mem, addr, t2);                                                \
+    ATOMIC_ALIGNMENT_CHECK(addr, t2);                                       \
+    t2 expected_wrapped = (t2)expected;                                     \
+    t2 replacement_wrapped = (t2)replacement;                               \
+    t2 ret;                                                                 \
+    wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2));      \
+    if (ret == expected_wrapped) {                                          \
+      wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \
+                     sizeof(t2));                                           \
+    }                                                                       \
+    return (t1)expected_wrapped;                                            \
+  }                                                                         \
+  static inline t1 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr,    \
+                                 t1 expected, t1 replacement) {             \
+    MEMCHECK(mem, addr, t2);                                                \
+    ATOMIC_ALIGNMENT_CHECK(addr, t2);                                       \
+    t2 expected_wrapped = (t2)expected;                                     \
+    t2 replacement_wrapped = (t2)replacement;                               \
+    atomic_compare_exchange_strong(                                         \
+        (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)),              \
+        &expected_wrapped, replacement_wrapped);                            \
+    return (t1)expected_wrapped;                                            \
   }
 
 DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8);
@@ -237,3 +231,5 @@ DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw8_cmpxchg_u, u64, u8);
 DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw16_cmpxchg_u, u64, u16);
 DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw32_cmpxchg_u, u64, u32);
 DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw_cmpxchg, u64, u64);
+
+#define atomic_fence() atomic_thread_fence(memory_order_seq_cst)