Skip tests that fail on windows and enable all the rest (#3035)

This lets us run most tests at least on that platform.

Add a new function for skipping those tests, skip_if_on_windows,
so that it's easy to find which tests are disabled on windows for later fixing
efforts.

This fixes a few minor issues for windows, like comparisons
should ignore \r in some cases.

Rename all passes tests that use --dwarfdump to contain "dwarf"
in their name, which makes it easy to skip those (and is clearer
anyhow).

32 files changed, 47 insertions, 14 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 046e60ed4..6e84258f2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -92,9 +92,6 @@ jobs:
 
     - name: test
       run: python check.py --binaryen-bin=out/install/bin
-      # Currently disabled on windows due to a single test failure.
-      # https://github.com/WebAssembly/binaryen/issues/2781
-      if: matrix.os != 'windows-latest'
 
   build-clang:
     name: clang
diff --git a/check.py b/check.py
index c5524481f..a47d4ef7e 100755
--- a/check.py
+++ b/check.py
@@ -190,7 +190,11 @@ def run_spec_tests():
     print('\n[ checking wasm-shell spec testcases... ]\n')
 
     for wast in shared.options.spec_tests:
-        print('..', os.path.basename(wast))
+        base = os.path.basename(wast)
+        print('..', base)
+        # windows has some failures that need to be investigated
+        if base == 'names.wast' and shared.skip_if_on_windows('spec: ' + base):
+            continue
 
         def run_spec_test(wast):
             cmd = shared.WASM_SHELL + [wast]
@@ -214,13 +218,13 @@ def run_spec_tests():
                 if actual != expected:
                     shared.fail(actual, expected)
 
-        expected = os.path.join(shared.get_test_dir('spec'), 'expected-output', os.path.basename(wast) + '.log')
+        expected = os.path.join(shared.get_test_dir('spec'), 'expected-output', base + '.log')
 
         # some spec tests should fail (actual process failure, not just assert_invalid)
         try:
             actual = run_spec_test(wast)
         except Exception as e:
-            if ('wasm-validator error' in str(e) or 'parse exception' in str(e)) and '.fail.' in os.path.basename(wast):
+            if ('wasm-validator error' in str(e) or 'parse exception' in str(e)) and '.fail.' in base:
                 print('<< test failed as expected >>')
                 continue  # don't try all the binary format stuff TODO
             else:
@@ -229,7 +233,7 @@ def run_spec_tests():
         check_expected(actual, expected)
 
         # skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature
-        if 'exports.wast' in os.path.basename(wast):  # FIXME
+        if 'exports.wast' in base:  # FIXME
             continue
 
         # check binary format. here we can verify execution of the final
@@ -240,7 +244,7 @@ def run_spec_tests():
 
         # FIXME Remove reference type tests from this list after nullref is
         # implemented in V8
-        if os.path.basename(wast) not in ['comments.wast', 'ref_null.wast', 'ref_is_null.wast', 'ref_func.wast', 'old_select.wast']:
+        if base not in ['comments.wast', 'ref_null.wast', 'ref_is_null.wast', 'ref_func.wast', 'old_select.wast']:
             split_num = 0
             actual = ''
             for module, asserts in support.split_wast(wast):
@@ -254,7 +258,7 @@ def run_spec_tests():
                 open(result_wast, 'a').write('\n' + '\n'.join(asserts))
                 actual += run_spec_test(result_wast)
             # compare all the outputs to the expected output
-            check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', os.path.basename(wast) + '.log'))
+            check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', base + '.log'))
         else:
             # handle unsplittable wast files
             run_spec_test(wast)
@@ -282,6 +286,9 @@ def run_gcc_tests():
     if not shared.NATIVECC or not shared.NATIVEXX:
         shared.fail_with_error('Native compiler (e.g. gcc/g++) was not found in PATH!')
         return
+    # windows + gcc will need some work
+    if shared.skip_if_on_windows('gcc'):
+        return
 
     for t in sorted(os.listdir(shared.get_test_dir('example'))):
         output_file = 'example'
@@ -332,6 +339,10 @@ def run_gcc_tests():
 def run_unittest():
     print('\n[ checking unit tests...]\n')
 
+    # windows has some failures that need to be investigated
+    if shared.skip_if_on_windows('unit'):
+        return
+
     # equivalent to `python -m unittest discover -s ./test -v`
     suite = unittest.defaultTestLoader.discover(os.path.dirname(shared.options.binaryen_test))
     result = unittest.TextTestRunner(verbosity=2, failfast=shared.options.abort_on_first_failure).run(suite)
diff --git a/scripts/test/shared.py b/scripts/test/shared.py
index d677c33a6..b0cd72d17 100644
--- a/scripts/test/shared.py
+++ b/scripts/test/shared.py
@@ -376,13 +376,13 @@ else:
 
 # 11/27/2019: We updated the spec test suite to upstream spec repo. For some
 # files that started failing after this update, we added the new files to this
-# blacklist and preserved old ones by renaming them to 'old_[FILENAME].wast'
+# skip-list and preserved old ones by renaming them to 'old_[FILENAME].wast'
 # not to lose coverage. When the cause of the error is fixed or the unsupported
 # construct gets support so the new test passes, we can delete the
 # corresponding 'old_[FILENAME].wast' file. When you fix the new file and
 # delete the old file, make sure you rename the corresponding .wast.log file in
 # expected-output/ if any.
-SPEC_TEST_BLACKLIST = [
+SPEC_TESTS_TO_SKIP = [
     # Stacky code / notation
     'block.wast',
     'call.wast',
@@ -437,7 +437,7 @@ SPEC_TEST_BLACKLIST = [
     'unreached-invalid.wast'  # 'assert_invalid' failure
 ]
 options.spec_tests = [t for t in options.spec_tests if os.path.basename(t) not
-                      in SPEC_TEST_BLACKLIST]
+                      in SPEC_TESTS_TO_SKIP]
 
 
 # check utilities
@@ -507,3 +507,13 @@ def with_pass_debug(check):
         else:
             if 'BINARYEN_PASS_DEBUG' in os.environ:
                 del os.environ['BINARYEN_PASS_DEBUG']
+
+
+# checks if we are on windows, and if so logs out that a test is being skipped,
+# and returns True. This is a central location for all test skipping on
+# windows, so that we can easily find which tests are skipped.
+def skip_if_on_windows(name):
+    if get_platform() == 'windows':
+        print('skipping test "%s" on windows' % name)
+        return True
+    return False
diff --git a/scripts/test/wasm2js.py b/scripts/test/wasm2js.py
index 3ccf3c9dc..5ae146d19 100644
--- a/scripts/test/wasm2js.py
+++ b/scripts/test/wasm2js.py
@@ -121,6 +121,8 @@ def test_asserts_output():
 
 def test_wasm2js():
     print('\n[ checking wasm2js testcases... ]\n')
+    if shared.skip_if_on_windows('wasm2js'):
+        return
     test_wasm2js_output()
     test_asserts_output()
 
diff --git a/scripts/test/wasm_opt.py b/scripts/test/wasm_opt.py
index 8c6bbdec5..986ce2f61 100644
--- a/scripts/test/wasm_opt.py
+++ b/scripts/test/wasm_opt.py
@@ -45,6 +45,12 @@ def test_wasm_opt():
 
     for t in shared.get_tests(shared.get_test_dir('passes'), ['.wast', '.wasm']):
         print('..', os.path.basename(t))
+        # windows has some failures that need to be investigated:
+        # * ttf tests have different outputs - order of execution of params?
+        # * dwarf tests print windows slashes instead of unix
+        if ('translate-to-fuzz' in t or 'dwarf' in t) and \
+           shared.skip_if_on_windows('fuzz translation tests'):
+            continue
         binary = '.wasm' in t
         base = os.path.basename(t).replace('.wast', '').replace('.wasm', '')
         passname = base
diff --git a/test/passes/fannkuch0.bin.txt b/test/passes/fannkuch0_dwarf.bin.txt
index 7e225f9b0..7e225f9b0 100644
--- a/test/passes/fannkuch0.bin.txt
+++ b/test/passes/fannkuch0_dwarf.bin.txt
diff --git a/test/passes/fannkuch0.passes b/test/passes/fannkuch0_dwarf.passes
index edbfb02ae..edbfb02ae 100644
--- a/test/passes/fannkuch0.passes
+++ b/test/passes/fannkuch0_dwarf.passes
diff --git a/test/passes/fannkuch0.wasm b/test/passes/fannkuch0_dwarf.wasm
index ab40bedc4..ab40bedc4 100644
--- a/test/passes/fannkuch0.wasm
+++ b/test/passes/fannkuch0_dwarf.wasm
diff --git a/test/passes/fannkuch3.bin.txt b/test/passes/fannkuch3_dwarf.bin.txt
index 658e2e0a9..658e2e0a9 100644
--- a/test/passes/fannkuch3.bin.txt
+++ b/test/passes/fannkuch3_dwarf.bin.txt
diff --git a/test/passes/fannkuch3.passes b/test/passes/fannkuch3_dwarf.passes
index edbfb02ae..edbfb02ae 100644
--- a/test/passes/fannkuch3.passes
+++ b/test/passes/fannkuch3_dwarf.passes
diff --git a/test/passes/fannkuch3.wasm b/test/passes/fannkuch3_dwarf.wasm
index 54a06c057..54a06c057 100644
--- a/test/passes/fannkuch3.wasm
+++ b/test/passes/fannkuch3_dwarf.wasm
diff --git a/test/passes/fannkuch3_manyopts.bin.txt b/test/passes/fannkuch3_manyopts_dwarf.bin.txt
index e9029b6b8..e9029b6b8 100644
--- a/test/passes/fannkuch3_manyopts.bin.txt
+++ b/test/passes/fannkuch3_manyopts_dwarf.bin.txt
diff --git a/test/passes/fannkuch3_manyopts.passes b/test/passes/fannkuch3_manyopts_dwarf.passes
index 8324d57aa..8324d57aa 100644
--- a/test/passes/fannkuch3_manyopts.passes
+++ b/test/passes/fannkuch3_manyopts_dwarf.passes
diff --git a/test/passes/fannkuch3_manyopts.wasm b/test/passes/fannkuch3_manyopts_dwarf.wasm
index 54a06c057..54a06c057 100644
--- a/test/passes/fannkuch3_manyopts.wasm
+++ b/test/passes/fannkuch3_manyopts_dwarf.wasm
diff --git a/test/passes/fib2.bin.txt b/test/passes/fib2_dwarf.bin.txt
index ac0c26cf5..ac0c26cf5 100644
--- a/test/passes/fib2.bin.txt
+++ b/test/passes/fib2_dwarf.bin.txt
diff --git a/test/passes/fib2.passes b/test/passes/fib2_dwarf.passes
index edbfb02ae..edbfb02ae 100644
--- a/test/passes/fib2.passes
+++ b/test/passes/fib2_dwarf.passes
diff --git a/test/passes/fib2.wasm b/test/passes/fib2_dwarf.wasm
index 01314643e..01314643e 100644
--- a/test/passes/fib2.wasm
+++ b/test/passes/fib2_dwarf.wasm
diff --git a/test/passes/fib2_emptylocspan.bin.txt b/test/passes/fib2_emptylocspan_dwarf.bin.txt
index 21f2fb878..21f2fb878 100644
--- a/test/passes/fib2_emptylocspan.bin.txt
+++ b/test/passes/fib2_emptylocspan_dwarf.bin.txt
diff --git a/test/passes/fib2_emptylocspan.passes b/test/passes/fib2_emptylocspan_dwarf.passes
index edbfb02ae..edbfb02ae 100644
--- a/test/passes/fib2_emptylocspan.passes
+++ b/test/passes/fib2_emptylocspan_dwarf.passes
diff --git a/test/passes/fib2_emptylocspan.wasm b/test/passes/fib2_emptylocspan_dwarf.wasm
index 94e6cb5b9..94e6cb5b9 100644
--- a/test/passes/fib2_emptylocspan.wasm
+++ b/test/passes/fib2_emptylocspan_dwarf.wasm
diff --git a/test/passes/fib_nonzero-low-pc.bin.txt b/test/passes/fib_nonzero-low-pc_dwarf.bin.txt
index ac28b21a4..ac28b21a4 100644
--- a/test/passes/fib_nonzero-low-pc.bin.txt
+++ b/test/passes/fib_nonzero-low-pc_dwarf.bin.txt
diff --git a/test/passes/fib_nonzero-low-pc.passes b/test/passes/fib_nonzero-low-pc_dwarf.passes
index 523725b1c..523725b1c 100644
--- a/test/passes/fib_nonzero-low-pc.passes
+++ b/test/passes/fib_nonzero-low-pc_dwarf.passes
diff --git a/test/passes/fib_nonzero-low-pc.wasm b/test/passes/fib_nonzero-low-pc_dwarf.wasm
index 5f44336d4..5f44336d4 100644
--- a/test/passes/fib_nonzero-low-pc.wasm
+++ b/test/passes/fib_nonzero-low-pc_dwarf.wasm
diff --git a/test/passes/ignore_missing_func.bin.txt b/test/passes/ignore_missing_func_dwarf.bin.txt
index c5f45cfdc..c5f45cfdc 100644
--- a/test/passes/ignore_missing_func.bin.txt
+++ b/test/passes/ignore_missing_func_dwarf.bin.txt
diff --git a/test/passes/ignore_missing_func.passes b/test/passes/ignore_missing_func_dwarf.passes
index a85a46b5e..a85a46b5e 100644
--- a/test/passes/ignore_missing_func.passes
+++ b/test/passes/ignore_missing_func_dwarf.passes
diff --git a/test/passes/ignore_missing_func.wasm b/test/passes/ignore_missing_func_dwarf.wasm
index 68c6de155..68c6de155 100644
--- a/test/passes/ignore_missing_func.wasm
+++ b/test/passes/ignore_missing_func_dwarf.wasm
diff --git a/test/passes/multi_line_table.bin.txt b/test/passes/multi_line_table_dwarf.bin.txt
index 84305a175..84305a175 100644
--- a/test/passes/multi_line_table.bin.txt
+++ b/test/passes/multi_line_table_dwarf.bin.txt
diff --git a/test/passes/multi_line_table.passes b/test/passes/multi_line_table_dwarf.passes
index 9e52f4d0c..9e52f4d0c 100644
--- a/test/passes/multi_line_table.passes
+++ b/test/passes/multi_line_table_dwarf.passes
diff --git a/test/passes/multi_line_table.wasm b/test/passes/multi_line_table_dwarf.wasm
index 312c013da..312c013da 100644
--- a/test/passes/multi_line_table.wasm
+++ b/test/passes/multi_line_table_dwarf.wasm
diff --git a/test/unit/test_asyncify.py b/test/unit/test_asyncify.py
index 55b198cbb..1010c07c4 100644
--- a/test/unit/test_asyncify.py
+++ b/test/unit/test_asyncify.py
@@ -29,7 +29,7 @@ class AsyncifyTest(utils.BinaryenTestCase):
             shared.run_process(shared.WASM_DIS + ['a.wasm', '-o', 'a.wat'])
             output = shared.run_process(shared.WASM_SHELL + ['a.wat'], capture_output=True).stdout
             with open(self.input_path('asyncify-pure.txt'), 'r') as f:
-                self.assertEqual(f.read(), output)
+                self.assert_equal_ignoring_line_endings(f.read(), output)
 
         # test wat input
         wat = self.input_path('asyncify-pure.wat')
diff --git a/test/unit/test_features.py b/test/unit/test_features.py
index 3b30595d6..506fd8809 100644
--- a/test/unit/test_features.py
+++ b/test/unit/test_features.py
@@ -354,4 +354,4 @@ class TargetFeaturesSectionTest(utils.BinaryenTestCase):
             '--enable-tail-call',
             '--enable-reference-types',
             '--enable-multivalue'
-        ], p2.stdout.split())
+        ], p2.stdout.splitlines())
diff --git a/test/unit/utils.py b/test/unit/utils.py
index a9bfc3740..d91141e36 100644
--- a/test/unit/utils.py
+++ b/test/unit/utils.py
@@ -36,3 +36,10 @@ class BinaryenTestCase(unittest.TestCase):
         self.assertEqual(p.stderr, '')
         self.assertEqual(p.stdout.split('\n')[:-1],
                          ['--enable-' + f for f in features])
+
+    # similar to assertEqual, but while ignoring line ending differences such
+    # as those between windows and unix
+    def assert_equal_ignoring_line_endings(self, left, right):
+        left = left.replace('\r\n', '\n')
+        right = right.replace('\r\n', '\n')
+        self.assertEqual(left, right)