afl-fuzz fixes for table/memory instantiation (#1460)

* replace assert with a proper trap for an invalid offset in table initialization

* fix offset handling in initial table size computation: it is an unsigned value

* handle traps in fuzz-exec when creating instance

* optimization may remove imports - and imported table init may trap, so opts may remove that trap. check for result comparisons in the right order, so we don't get bothered by that

2 files changed, 27 insertions, 19 deletions

diff --git a/src/shell-interface.h b/src/shell-interface.h
index e41d9de24..0a98ce05a 100644
--- a/src/shell-interface.h
+++ b/src/shell-interface.h
@@ -95,8 +95,10 @@ struct ShellExternalInterface final : ModuleInstance::ExternalInterface {
     memory.resize(wasm.memory.initial * wasm::Memory::kPageSize);
     // apply memory segments
     for (auto& segment : wasm.memory.segments) {
-      Address offset = ConstantExpressionRunner<TrivialGlobalManager>(instance.globals).visit(segment.offset).value.geti32();
-      assert(offset + segment.data.size() <= wasm.memory.initial * wasm::Memory::kPageSize);
+      Address offset = (uint32_t)ConstantExpressionRunner<TrivialGlobalManager>(instance.globals).visit(segment.offset).value.geti32();
+      if (offset + segment.data.size() > wasm.memory.initial * wasm::Memory::kPageSize) {
+        trap("invalid offset when initializing memory");
+      }
       for (size_t i = 0; i != segment.data.size(); ++i) {
         memory.set(offset + i, segment.data[i]);
       }
@@ -104,8 +106,10 @@ struct ShellExternalInterface final : ModuleInstance::ExternalInterface {
 
     table.resize(wasm.table.initial);
     for (auto& segment : wasm.table.segments) {
-      Address offset = ConstantExpressionRunner<TrivialGlobalManager>(instance.globals).visit(segment.offset).value.geti32();
-      assert(offset + segment.data.size() <= wasm.table.initial);
+      Address offset = (uint32_t)ConstantExpressionRunner<TrivialGlobalManager>(instance.globals).visit(segment.offset).value.geti32();
+      if (offset + segment.data.size() > wasm.table.initial) {
+        trap("invalid offset when initializing table");
+      }
       for (size_t i = 0; i != segment.data.size(); ++i) {
         table[offset + i] = segment.data[i];
       }
diff --git a/src/tools/execution-results.h b/src/tools/execution-results.h
index 7537d9911..fe82c024a 100644
--- a/src/tools/execution-results.h
+++ b/src/tools/execution-results.h
@@ -37,20 +37,24 @@ struct ExecutionResults {
       return;
     }
     ShellExternalInterface interface;
-    ModuleInstance instance(wasm, &interface);
-    // execute all exported methods (that are therefore preserved through opts)
-    for (auto& exp : wasm.exports) {
-      if (exp->kind != ExternalKind::Function) continue;
-      auto* func = wasm.getFunction(exp->value);
-      if (func->result != none) {
-        // this has a result
-        results[exp->name] = run(func, wasm, instance);
-        std::cout << "[fuzz-exec] note result: " << exp->name << " => " << results[exp->name] << '\n';
-      } else {
-        // no result, run it anyhow (it might modify memory etc.)
-        run(func, wasm, instance);
-        std::cout << "[fuzz-exec] no result for void func: " << exp->name << '\n';
+    try {
+      ModuleInstance instance(wasm, &interface);
+      // execute all exported methods (that are therefore preserved through opts)
+      for (auto& exp : wasm.exports) {
+        if (exp->kind != ExternalKind::Function) continue;
+        auto* func = wasm.getFunction(exp->value);
+        if (func->result != none) {
+          // this has a result
+          results[exp->name] = run(func, wasm, instance);
+          std::cout << "[fuzz-exec] note result: " << exp->name << " => " << results[exp->name] << '\n';
+        } else {
+          // no result, run it anyhow (it might modify memory etc.)
+          run(func, wasm, instance);
+          std::cout << "[fuzz-exec] no result for void func: " << exp->name << '\n';
+        }
       }
+    } catch (const TrapException&) {
+      // may throw in instance creation (init of offsets)
     }
     std::cout << "[fuzz-exec] " << results.size() << " results noted\n";
   }
@@ -67,9 +71,9 @@ struct ExecutionResults {
   }
 
   bool operator==(ExecutionResults& other) {
-    for (auto& iter : results) {
+    for (auto& iter : other.results) {
       auto name = iter.first;
-      if (other.results.find(name) == other.results.end()) {
+      if (results.find(name) == results.end()) {
         std::cout << "[fuzz-exec] missing " << name << '\n';
         abort();
       }