Improve testing on Windows (#3142)

This PR contains:
- Changes that enable/disable tests on Windows to allow for better local testing.
- Also changes many abort() into Fatal() when it is really just exiting on error. This is because abort() generates a dialog window on Windows which is not great in automated scripts.
- Improvements to CMake to better work with the project in IDEs (VS).

16 files changed, 55 insertions, 57 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9b8bac04d..d068d11fc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -145,6 +145,8 @@ if(MSVC)
   add_compile_flag("/wd4244")
   # 4722 warns that destructors never return, even with WASM_NORETURN.
   add_compile_flag("/wd4722")
+  # "destructor was implicitly defined as deleted" caused by LLVM headers.
+  add_compile_flag("/wd4624")
   add_compile_flag("/WX-")
   add_debug_compile_flag("/Od")
   add_nondebug_compile_flag("/O2")
@@ -156,6 +158,9 @@ if(MSVC)
   # Don't warn about using "strdup" as a reserved name.
   add_compile_flag("/D_CRT_NONSTDC_NO_DEPRECATE")
 
+  # multi-core build.
+  add_compile_flag("/MP")
+
   if(BYN_ENABLE_ASSERTIONS)
     # On non-Debug builds cmake automatically defines NDEBUG, so we
     # explicitly undefine it:
@@ -294,8 +299,10 @@ ENDIF()
 
 # Sources.
 
+file(GLOB binaryen_HEADERS src/*.h)
 set(binaryen_SOURCES
   src/binaryen-c.cpp
+  ${binaryen_HEADERS}
 )
 if(BUILD_STATIC_LIB)
   message(STATUS "Building libbinaryen as statically linked library.")
@@ -342,6 +349,7 @@ binaryen_add_executable(wasm-reduce src/tools/wasm-reduce.cpp)
 if(EMSCRIPTEN)
   set(binaryen_emscripten_SOURCES
     src/binaryen-c.cpp
+    ${binaryen_HEADERS}
   )
 
   # binaryen.js WebAssembly variant
diff --git a/auto_update_tests.py b/auto_update_tests.py
index 465518515..ef96072de 100755
--- a/auto_update_tests.py
+++ b/auto_update_tests.py
@@ -50,6 +50,9 @@ def update_example_tests():
             expected = os.path.splitext(t)[0] + '.txt'
         if not src.endswith(('.c', '.cpp')):
             continue
+        # windows + gcc will need some work
+        if shared.skip_if_on_windows('gcc'):
+            return
         # build the C file separately
         extra = [os.environ.get('CC') or 'gcc',
                  src, '-c', '-o', 'example.o',
@@ -120,8 +123,6 @@ def update_metadce_tests():
 
 
 def update_reduce_tests():
-    if not shared.has_shell_timeout():
-        return
     print('\n[ checking wasm-reduce ]\n')
     for t in shared.get_tests(shared.get_test_dir('reduce'), ['.wast']):
         print('..', os.path.basename(t))
diff --git a/check.py b/check.py
index 33a916574..52ab3c4de 100755
--- a/check.py
+++ b/check.py
@@ -338,10 +338,6 @@ def run_gcc_tests():
 def run_unittest():
     print('\n[ checking unit tests...]\n')
 
-    # windows has some failures that need to be investigated
-    if shared.skip_if_on_windows('unit'):
-        return
-
     # equivalent to `python -m unittest discover -s ./test -v`
     suite = unittest.defaultTestLoader.discover(os.path.dirname(shared.options.binaryen_test))
     result = unittest.TextTestRunner(verbosity=2, failfast=shared.options.abort_on_first_failure).run(suite)
diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py
index 3f79ec565..eddb33b21 100755
--- a/scripts/fuzz_opt.py
+++ b/scripts/fuzz_opt.py
@@ -916,7 +916,12 @@ if __name__ == '__main__':
         mean = float(total_input_size) / counter
         mean_of_squares = float(total_input_size_squares) / counter
         stddev = math.sqrt(mean_of_squares - (mean ** 2))
-        print('ITERATION:', counter, 'seed:', seed, 'size:', input_size, '(mean:', str(mean) + ', stddev:', str(stddev) + ')', 'speed:', counter / (time.time() - start_time), 'iters/sec, ', total_wasm_size / (time.time() - start_time), 'wasm_bytes/sec\n')
+        elapsed = max(0.000001, time.time() - start_time)
+        print('ITERATION:', counter, 'seed:', seed, 'size:', input_size,
+              '(mean:', str(mean) + ', stddev:', str(stddev) + ')',
+              'speed:', counter / elapsed,
+              'iters/sec, ', total_wasm_size / elapsed,
+              'wasm_bytes/sec\n')
         with open(raw_input_data, 'wb') as f:
             f.write(bytes([random.randint(0, 255) for x in range(input_size)]))
         assert os.path.getsize(raw_input_data) == input_size
diff --git a/scripts/test/wasm_opt.py b/scripts/test/wasm_opt.py
index 2c7d49eb7..d6441bd8d 100644
--- a/scripts/test/wasm_opt.py
+++ b/scripts/test/wasm_opt.py
@@ -159,6 +159,12 @@ def update_wasm_opt_tests():
     print('\n[ checking wasm-opt passes... ]\n')
     for t in shared.get_tests(shared.get_test_dir('passes'), ['.wast', '.wasm']):
         print('..', os.path.basename(t))
+        # windows has some failures that need to be investigated:
+        # * ttf tests have different outputs - order of execution of params?
+        # * dwarf tests print windows slashes instead of unix
+        if ('translate-to-fuzz' in t or 'dwarf' in t) and \
+           shared.skip_if_on_windows('fuzz translation tests'):
+            continue
         binary = t.endswith('.wasm')
         base = os.path.basename(t).replace('.wast', '').replace('.wasm', '')
         passname = base
diff --git a/src/passes/ExtractFunction.cpp b/src/passes/ExtractFunction.cpp
index fbc1aa7c4..34fb449d4 100644
--- a/src/passes/ExtractFunction.cpp
+++ b/src/passes/ExtractFunction.cpp
@@ -42,8 +42,7 @@ struct ExtractFunction : public Pass {
       }
     }
     if (!found) {
-      std::cerr << "could not find the function to extract\n";
-      abort();
+      Fatal() << "could not find the function to extract\n";
     }
     // clear data
     module->memory.segments.clear();
diff --git a/src/passes/pass.cpp b/src/passes/pass.cpp
index 62c4c6452..48d121098 100644
--- a/src/passes/pass.cpp
+++ b/src/passes/pass.cpp
@@ -591,16 +591,15 @@ void PassRunner::run() {
         if (!WasmValidator().validate(*wasm, validationFlags)) {
           WasmPrinter::printModule(wasm);
           if (passDebug >= 2) {
-            std::cerr << "Last pass (" << pass->name
-                      << ") broke validation. Here is the module before: \n"
-                      << moduleBefore.str() << "\n";
+            Fatal() << "Last pass (" << pass->name
+                    << ") broke validation. Here is the module before: \n"
+                    << moduleBefore.str() << "\n";
           } else {
-            std::cerr << "Last pass (" << pass->name
-                      << ") broke validation. Run with BINARYEN_PASS_DEBUG=2 "
-                         "in the env to see the earlier state, or 3 to dump "
-                         "byn-* files for each pass\n";
+            Fatal() << "Last pass (" << pass->name
+                    << ") broke validation. Run with BINARYEN_PASS_DEBUG=2 "
+                       "in the env to see the earlier state, or 3 to dump "
+                       "byn-* files for each pass\n";
           }
-          abort();
         }
       }
       if (passDebug >= 3) {
@@ -613,8 +612,7 @@ void PassRunner::run() {
       std::cerr << "[PassRunner] (final validation)\n";
       if (!WasmValidator().validate(*wasm, validationFlags)) {
         WasmPrinter::printModule(wasm);
-        std::cerr << "final module does not validate\n";
-        abort();
+        Fatal() << "final module does not validate\n";
       }
     }
   } else {
diff --git a/src/tools/execution-results.h b/src/tools/execution-results.h
index 0881163f1..b63a276ae 100644
--- a/src/tools/execution-results.h
+++ b/src/tools/execution-results.h
@@ -99,7 +99,7 @@ struct ExecutionResults {
     optimizedResults.get(wasm);
     if (optimizedResults != *this) {
       std::cout << "[fuzz-exec] optimization passes changed execution results";
-      abort();
+      exit(1);
     }
   }
 
diff --git a/src/tools/wasm-as.cpp b/src/tools/wasm-as.cpp
index 3a907c4c3..be18ed355 100644
--- a/src/tools/wasm-as.cpp
+++ b/src/tools/wasm-as.cpp
@@ -54,9 +54,8 @@ int main(int argc, const char* argv[]) {
          Options::Arguments::One,
          [](Options* o, const std::string& argument) {
            if (argument != "web" && argument != "none" && argument != "wasm") {
-             std::cerr << "Valid arguments for --validate flag are 'wasm', "
-                          "'web', and 'none'.\n";
-             exit(1);
+             Fatal() << "Valid arguments for --validate flag are 'wasm', "
+                        "'web', and 'none'.\n";
            }
            o->extra["validate"] = argument;
          })
diff --git a/src/tools/wasm-opt.cpp b/src/tools/wasm-opt.cpp
index 24ea9c31c..ecf4f1c7c 100644
--- a/src/tools/wasm-opt.cpp
+++ b/src/tools/wasm-opt.cpp
@@ -377,8 +377,7 @@ int main(int argc, const char* argv[]) {
     auto secondOutput = runCommand(extraFuzzCommand);
     std::cout << "[extra-fuzz-command second output:]\n" << firstOutput << '\n';
     if (firstOutput != secondOutput) {
-      std::cerr << "extra fuzz command output differs\n";
-      abort();
+      Fatal() << "extra fuzz command output differs\n";
     }
   }
   return 0;
diff --git a/src/tools/wasm-shell.cpp b/src/tools/wasm-shell.cpp
index 8ba6a4142..69670192b 100644
--- a/src/tools/wasm-shell.cpp
+++ b/src/tools/wasm-shell.cpp
@@ -192,8 +192,7 @@ static void run_asserts(Name moduleName,
         Colors::red(std::cerr);
         std::cerr << "[should have been invalid]\n";
         Colors::normal(std::cerr);
-        std::cerr << &wasm << '\n';
-        abort();
+        Fatal() << &wasm << '\n';
       }
     } else if (id == INVOKE) {
       assert(wasm);
@@ -222,8 +221,7 @@ static void run_asserts(Name moduleName,
         }
         std::cerr << "seen " << result << ", expected " << expected << '\n';
         if (expected != result) {
-          std::cout << "unexpected, should be identical\n";
-          abort();
+          Fatal() << "unexpected, should be identical\n";
         }
       }
       if (id == ASSERT_TRAP) {
@@ -314,8 +312,8 @@ int main(int argc, const char* argv[]) {
         bool valid = WasmValidator().validate(*modules[moduleName]);
         if (!valid) {
           WasmPrinter::printModule(modules[moduleName].get());
+          Fatal() << "module failed to validate, see above";
         }
-        assert(valid);
         run_asserts(moduleName,
                     &i,
                     &checked,
@@ -329,7 +327,7 @@ int main(int argc, const char* argv[]) {
     }
   } catch (ParseException& p) {
     p.dump(std::cerr);
-    abort();
+    exit(1);
   }
 
   if (checked) {
diff --git a/src/tools/wasm2js.cpp b/src/tools/wasm2js.cpp
index 524be4c5c..68334708f 100644
--- a/src/tools/wasm2js.cpp
+++ b/src/tools/wasm2js.cpp
@@ -630,8 +630,7 @@ Ref AssertionEmitter::emitAssertReturnFunc(Builder& wasmBuilder,
       }
 
       default: {
-        std::cerr << "Unhandled type in assert: " << resType << std::endl;
-        abort();
+        Fatal() << "Unhandled type in assert: " << resType;
       }
     }
   } else {
diff --git a/src/wasm/CMakeLists.txt b/src/wasm/CMakeLists.txt
index 32cdf50c8..d24618c63 100644
--- a/src/wasm/CMakeLists.txt
+++ b/src/wasm/CMakeLists.txt
@@ -1,4 +1,4 @@
-FILE(GLOB wasm_HEADERS *.h)
+file(GLOB wasm_HEADERS ../*.h)
 set(wasm_SOURCES
   literal.cpp
   wasm.cpp
diff --git a/src/wasm2js.h b/src/wasm2js.h
index 8418d5b15..727e87379 100644
--- a/src/wasm2js.h
+++ b/src/wasm2js.h
@@ -1268,9 +1268,8 @@ Ref Wasm2JSBuilder::processFunctionBody(Module* m,
                                           ValueBuilder::makePtrShift(ptr, 2));
               break;
             default: {
-              std::cerr << "Unhandled number of bytes in i32 load: "
-                        << curr->bytes << std::endl;
-              abort();
+              Fatal() << "Unhandled number of bytes in i32 load: "
+                      << curr->bytes;
             }
           }
           break;
@@ -1284,8 +1283,7 @@ Ref Wasm2JSBuilder::processFunctionBody(Module* m,
                                       ValueBuilder::makePtrShift(ptr, 3));
           break;
         default: {
-          std::cerr << "Unhandled type in load: " << curr->type << std::endl;
-          abort();
+          Fatal() << "Unhandled type in load: " << curr->type;
         }
       }
       if (curr->isAtomic) {
@@ -1378,9 +1376,7 @@ Ref Wasm2JSBuilder::processFunctionBody(Module* m,
                                       ValueBuilder::makePtrShift(ptr, 3));
           break;
         default: {
-          std::cerr << "Unhandled type in store: " << curr->valueType
-                    << std::endl;
-          abort();
+          Fatal() << "Unhandled type in store: " << curr->valueType;
         }
       }
       if (curr->isAtomic) {
@@ -1431,7 +1427,7 @@ Ref Wasm2JSBuilder::processFunctionBody(Module* m,
             PLUS, ValueBuilder::makeDouble(curr->value.getf64()));
         }
         default:
-          abort();
+          Fatal() << "unknown const type";
       }
     }
 
@@ -1510,9 +1506,7 @@ Ref Wasm2JSBuilder::processFunctionBody(Module* m,
                 ValueBuilder::makeNum(16));
             }
             default: {
-              std::cerr << "Unhandled unary i32 operator: " << curr
-                        << std::endl;
-              abort();
+              Fatal() << "Unhandled unary i32 operator: " << curr;
             }
           }
         }
@@ -1607,8 +1601,7 @@ Ref Wasm2JSBuilder::processFunctionBody(Module* m,
           return ret;
         }
         default: {
-          std::cerr << "Unhandled type in unary: " << curr << std::endl;
-          abort();
+          Fatal() << "Unhandled type in unary: " << curr;
         }
       }
     }
@@ -1771,17 +1764,14 @@ Ref Wasm2JSBuilder::processFunctionBody(Module* m,
             case CopySignFloat32:
             case CopySignFloat64:
             default:
-              std::cerr << "Unhandled binary float operator: " << curr
-                        << std::endl;
-              abort();
+              Fatal() << "Unhandled binary float operator: ";
           }
           if (curr->type == Type::f32) {
             return makeAsmCoercion(ret, ASM_FLOAT);
           }
           return ret;
         default:
-          std::cerr << "Unhandled type in binary: " << curr << std::endl;
-          abort();
+          Fatal() << "Unhandled type in binary: " << curr;
       }
       return makeAsmCoercion(ret, wasmToAsmType(curr->type));
     }
@@ -2266,8 +2256,7 @@ void Wasm2JSGlue::emitPreES6() {
     // yet.
     if (baseModuleMap.count(base) && baseModuleMap[base] != module) {
       Fatal() << "the name " << base << " cannot be imported from "
-              << "two different modules yet\n";
-      abort();
+              << "two different modules yet";
     }
     baseModuleMap[base] = module;
 
diff --git a/test/unit/test_asyncify.py b/test/unit/test_asyncify.py
index 1010c07c4..7510c9492 100644
--- a/test/unit/test_asyncify.py
+++ b/test/unit/test_asyncify.py
@@ -14,7 +14,8 @@ class AsyncifyTest(utils.BinaryenTestCase):
             shared.run_process(shared.WASM_OPT + args + [self.input_path('asyncify-coroutine.wat'), '--asyncify', '-o', 'b.wasm'])
             shared.run_process(shared.WASM_OPT + args + [self.input_path('asyncify-stackOverflow.wat'), '--asyncify', '-o', 'c.wasm'])
             print('  file size: %d' % os.path.getsize('a.wasm'))
-            shared.run_process([shared.NODEJS, self.input_path('asyncify.js')])
+            if shared.NODEJS:
+                shared.run_process([shared.NODEJS, self.input_path('asyncify.js')])
 
         test(['-g'])
         test([])
diff --git a/test/unit/utils.py b/test/unit/utils.py
index d91141e36..b75ed311c 100644
--- a/test/unit/utils.py
+++ b/test/unit/utils.py
@@ -34,7 +34,7 @@ class BinaryenTestCase(unittest.TestCase):
         p = shared.run_process(cmd, check=False, capture_output=True)
         self.assertEqual(p.returncode, 0)
         self.assertEqual(p.stderr, '')
-        self.assertEqual(p.stdout.split('\n')[:-1],
+        self.assertEqual(p.stdout.splitlines(),
                          ['--enable-' + f for f in features])
 
     # similar to assertEqual, but while ignoring line ending differences such