Introduce lit/FileCheck tests (#3367)

lit and FileCheck are the tools used to run the majority of tests in LLVM. Each
lit test file contains the commands to be run for that test, so lit tests are
much more flexible and can be more precise than our current ad hoc testing
system. FileCheck reads expected test output from comments, so it allows test
output to be written alongside and interspersed with test input, making tests
more readable and precise than in our current system.

This PR adds a new suite to check.py that runs lit tests in the test/lit
directory. A few tests have been ported to demonstrate the features of the new
test runner.

This change is motivated by a need for greater flexibility in testing wasm-split.
See #3359.

9 files changed, 72 insertions, 91 deletions

diff --git a/test/lit/CMakeLists.txt b/test/lit/CMakeLists.txt
new file mode 100644
index 000000000..141896033
--- /dev/null
+++ b/test/lit/CMakeLists.txt
@@ -0,0 +1 @@
+configure_file(lit.site.cfg.py.in lit.site.cfg.py @ONLY)
diff --git a/test/lit/lit.cfg.py b/test/lit/lit.cfg.py
new file mode 100644
index 000000000..91dc1aeff
--- /dev/null
+++ b/test/lit/lit.cfg.py
@@ -0,0 +1,21 @@
+import lit.formats
+
+config.name = "Binaryen lit tests"
+config.test_format = lit.formats.ShTest(True)
+
+config.suffixes = ['.wat', '.wast']
+
+config.test_source_root = os.path.dirname(__file__)
+config.test_exec_root = os.path.join(config.src_root, 'out', 'test')
+
+# Replace all Binaryen tools with their absolute paths
+bin_dir = os.path.join(config.binaryen_root, 'bin')
+for tool_file in os.listdir(bin_dir):
+    tool_path = config.binaryen_root + '/bin/' + tool_file
+    tool = tool_file[:-4] if tool_file.endswith('.exe') else tool_file
+    config.substitutions.append((tool, tool_path))
+
+# Also make the `not` command available
+not_file = config.src_root + '/scripts/not.py'
+python = sys.executable.replace('\\', '/')
+config.substitutions.append(('not', python + ' ' + not_file))
diff --git a/test/lit/lit.site.cfg.py.in b/test/lit/lit.site.cfg.py.in
new file mode 100644
index 000000000..4b976dbad
--- /dev/null
+++ b/test/lit/lit.site.cfg.py.in
@@ -0,0 +1,7 @@
+import os
+
+config.src_root = r'@CMAKE_SOURCE_DIR@'
+config.binaryen_root = r'@CMAKE_BINARY_DIR@'
+
+lit_config.load_config(
+    config, os.path.join(config.src_root, 'test', 'lit', 'lit.cfg.py'))
diff --git a/test/lit/parse-error.wast b/test/lit/parse-error.wast
new file mode 100644
index 000000000..c301103b2
--- /dev/null
+++ b/test/lit/parse-error.wast
@@ -0,0 +1,10 @@
+;; Test that parse errors have helpful messages
+
+;; RUN: not wasm-opt %s 2>&1 | filecheck %s
+;; CHECK: [parse exception: abc (at 8:4)]
+
+(module
+  (func $foo
+    (abc)
+  )
+)
diff --git a/test/lit/validation/shared-memory.wast b/test/lit/validation/shared-memory.wast
new file mode 100644
index 000000000..259e89b81
--- /dev/null
+++ b/test/lit/validation/shared-memory.wast
@@ -0,0 +1,11 @@
+;; Test that shared memory requires atomics
+
+;; RUN: not wasm-opt %s 2>&1 | filecheck %s --check-prefix NO-ATOMICS
+;; RUN: wasm-opt %s --enable-threads -o - -S | filecheck %s --check-prefix ATOMICS
+
+;; NO-ATOMICS: memory is shared, but atomics are disabled
+;; ATOMICS: (memory $0 (shared 10 20))
+
+(module
+  (memory (shared 10 20))
+)
diff --git a/test/lit/wasm-emscripten-finalize/bigint.wat b/test/lit/wasm-emscripten-finalize/bigint.wat
new file mode 100644
index 000000000..817fc9dca
--- /dev/null
+++ b/test/lit/wasm-emscripten-finalize/bigint.wat
@@ -0,0 +1,22 @@
+;; Test that the --bigint option prevents i64s from being split up
+
+;; Run without --bigint to get a baseline
+;; RUN: wasm-emscripten-finalize %s -S | filecheck %s --check-prefix MVP
+
+;; Then run with --bigint to see the difference
+;; RUN: wasm-emscripten-finalize %s -S --bigint | filecheck %s --check-prefix BIGINT
+
+;; MVP: (export "dynCall_jj" (func $legalstub$dynCall_jj))
+;; MVP: (func $legalstub$dynCall_jj (param $0 i32) (param $1 i32) (param $2 i32) (result i32)
+
+;; BIGINT-NOT: legalstub
+;; BIGINT: (export "dynCall_jj" (func $dynCall_jj))
+;; BIGINT: (func $dynCall_jj (param $fptr i32) (param $0 i64) (result i64)
+
+(module
+ (table $0 1 1 funcref)
+ (elem (i32.const 1) $foo)
+ (func $foo (param i64) (result i64)
+  (unreachable)
+ )
+)
diff --git a/test/lld/bigint.wat b/test/lld/bigint.wat
deleted file mode 100644
index 4589f396e..000000000
--- a/test/lld/bigint.wat
+++ /dev/null
@@ -1,10 +0,0 @@
-(module
- (table $0 1 1 funcref)
- (elem (i32.const 1) $foo)
- (export "__data_end" (global $global$1))
- (global $global$0 (mut i32) (i32.const 66208))
- (global $global$1 i32 (i32.const 658))
- (func $foo (param i64) (result i64)
-  (unreachable)
- )
-)
diff --git a/test/lld/bigint.wat.out b/test/lld/bigint.wat.out
deleted file mode 100644
index bc966d4e7..000000000
--- a/test/lld/bigint.wat.out
+++ /dev/null
@@ -1,41 +0,0 @@
-(module
- (type $i64_=>_i64 (func (param i64) (result i64)))
- (type $i32_i64_=>_i64 (func (param i32 i64) (result i64)))
- (table $0 1 1 funcref)
- (elem (i32.const 1) $foo)
- (global $global$0 (mut i32) (i32.const 66208))
- (global $global$1 i32 (i32.const 658))
- (export "__data_end" (global $global$1))
- (export "dynCall_jj" (func $dynCall_jj))
- (func $foo (param $0 i64) (result i64)
-  (unreachable)
- )
- (func $dynCall_jj (param $fptr i32) (param $0 i64) (result i64)
-  (call_indirect (type $i64_=>_i64)
-   (local.get $0)
-   (local.get $fptr)
-  )
- )
-)
-(;
---BEGIN METADATA --
-{
-  "tableSize": 1,
-  "declares": [
-  ],
-  "externs": [
-  ],
-  "exports": [
-    "dynCall_jj"
-  ],
-  "namedGlobals": {
-    "__data_end" : "658"
-  },
-  "invokeFuncs": [
-  ],
-  "mainReadsParams": 0,
-  "features": [
-  ]
-}
--- END METADATA --
-;)
diff --git a/test/unit/test_errors.py b/test/unit/test_errors.py
deleted file mode 100644
index 15c5a44e6..000000000
--- a/test/unit/test_errors.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import os
-
-from scripts.test import shared
-from . import utils
-
-
-class ErrorsTest(utils.BinaryenTestCase):
-    def test_parsing_error_msg(self):
-        module = '''
-(module
-  (func $foo
-    (abc)
-  )
-)
-'''
-        p = shared.run_process(shared.WASM_OPT + ['--print', '-o', os.devnull],
-                               input=module, check=False, capture_output=True)
-        self.assertNotEqual(p.returncode, 0)
-        self.assertIn("parse exception: abc (at 4:4)", p.stderr)
-
-    def test_validation_error_msg(self):
-        def test(args=[], extra_expected=None):
-            module = '''
-(module
-  (memory (shared 10 20))
-)
-'''
-            p = shared.run_process(shared.WASM_OPT + ['-o', os.devnull] + args,
-                                   input=module, check=False, capture_output=True)
-            self.assertNotEqual(p.returncode, 0)
-            self.assertIn('memory is shared, but atomics are disabled', p.stderr)
-            if extra_expected:
-                self.assertIn(extra_expected, p.stdout)
-
-        test()
-        # when the user asks to print the module, we print it even if it is
-        # invalid, for debugging (otherwise, an invalid module would not reach
-        # the stage of runnning passes, and print is a pass, so nothing would
-        # be printed)
-        test(['--print'], '(module')