From 24d274983df9f7dbeebe8a890297d4f30d5bbca7 Mon Sep 17 00:00:00 2001
From: Heejin Ahn <aheejin@gmail.com>
Date: Fri, 29 Nov 2019 18:33:41 -0800
Subject: Update spec test suite (#2484)

This updates spec test suite to that of the current up-to-date version
of https://github.com/WebAssembly/spec repo.

- All failing tests are added in `BLACKLIST` in shared.py with reasons.
 - For tests that already existed and was passing and started failing
   after the update, we add the new test to the blacklist and preserve
   the old file by renaming it to 'old_[FILENAME].wast' not to lose test
   coverage. When the cause of the error is fixed or the unsupported
   construct gets support so the new test passes, we can delete the
   corresponding 'old_[FILENAME].wast' file.
- Adds support for `spectest.print_[type] style imports.
---
 scripts/test/shared.py   | 65 ++++++++++++++++++++++++++++++++++++++++++++----
 scripts/test/spectest.js | 21 ++++++++++++++--
 scripts/test/wasm2js.py  |  2 +-
 3 files changed, 80 insertions(+), 8 deletions(-)

(limited to 'scripts/test')

diff --git a/scripts/test/shared.py b/scripts/test/shared.py
index 9d1fa094e..9ba94d574 100644
--- a/scripts/test/shared.py
+++ b/scripts/test/shared.py
@@ -402,12 +402,67 @@ if not options.spec_tests:
 else:
     options.spec_tests = options.spec_tests[:]
 
+# 11/27/2019: We updated the spec test suite to upstream spec repo. For some
+# files that started failing after this update, we added the new files to this
+# blacklist and preserved old ones by renaming them to 'old_[FILENAME].wast'
+# not to lose coverage. When the cause of the error is fixed or the unsupported
+# construct gets support so the new test passes, we can delete the
+# corresponding 'old_[FILENAME].wast' file. When you fix the new file and
+# delete the old file, make sure you rename the corresponding .wast.log file in
+# expected-output/ if any.
 SPEC_TEST_BLACKLIST = [
-    'binary.wast',   # Cannot parse binary modules
-    'linking.wast',  # No support for 'register' command
-    'nop.wast',      # Stacky code
-    'stack.wast',    # Stacky code
-    'unwind.wast'    # Stacky code
+    # Stacky code / notation
+    'block.wast',
+    'call.wast',
+    'float_exprs.wast',
+    'globals.wast',
+    'loop.wast',
+    'nop.wast',
+    'select.wast',
+    'stack.wast',
+    'unwind.wast',
+
+    # Binary module
+    'binary.wast',
+    'binary-leb128.wast',
+    'custom.wast',
+
+    # Empty 'then' or 'else' in 'if'
+    'if.wast',
+    'local_set.wast',
+    'store.wast',
+
+    # No module in a file
+    'token.wast',
+    'utf8-custom-section-id.wast',
+    'utf8-import-field.wast',
+    'utf8-import-module.wast',
+    'utf8-invalid-encoding.wast',
+
+    # 'register' command
+    'imports.wast',
+    'linking.wast',
+
+    # Misc. unsupported constructs
+    'call_indirect.wast',  # Empty (param) and (result)
+    'const.wast',  # Unparenthesized expression
+    'data.wast',  # Various unsupported (data) notations
+    'elem.wast',  # Unsupported 'offset' syntax in (elem)
+    'exports.wast',  # Multiple inlined exports for a function
+    'func.wast',  # Forward named type reference
+    'skip-stack-guard-page.wast',  # Hexadecimal style (0x..) in memory offset
+
+    # Untriaged: We don't know the cause of the error yet
+    'address.wast',  # wasm2js 'assert_return' failure
+    'br_if.wast',  # Validation error
+    'float_literals.wast',  # 'assert_return' failure
+    'int_literals.wast',  # 'assert_return' failure
+    'local_tee.wast',  # Validation failure
+    'memory_grow.wast',  # 'assert_return' failure
+    'start.wast',  # Assertion failure
+    'type.wast',  # 'assertion_invalid' failure
+    'unreachable.wast',  # Validation failure
+    'unreached-invalid.wast'  # 'assert_invalid' failure
 ]
 options.spec_tests = [t for t in options.spec_tests if os.path.basename(t) not
                       in SPEC_TEST_BLACKLIST]
diff --git a/scripts/test/spectest.js b/scripts/test/spectest.js
index 0756209b6..3c4bac1b5 100644
--- a/scripts/test/spectest.js
+++ b/scripts/test/spectest.js
@@ -1,3 +1,20 @@
-export function print(...args) {
-  console.log(...args);
+export function print() {
+  console.log();
+}
+export function print_i32(arg) {
+  console.log(arg, ' : i32');
+}
+export function print_f32(arg) {
+  console.log(arg, ' : f32');
+}
+export function print_f64(arg) {
+  console.log(arg, ' : f64');
+}
+export function print_i32_f32(arg0, arg1) {
+  console.log(arg0, ' : i32');
+  console.log(arg1, ' : f32');
+}
+export function print_f64_f64(arg0, arg1) {
+  console.log(arg0, ' : f64');
+  console.log(arg1, ' : f64');
 }
diff --git a/scripts/test/wasm2js.py b/scripts/test/wasm2js.py
index a5e4ce6eb..31df227cf 100755
--- a/scripts/test/wasm2js.py
+++ b/scripts/test/wasm2js.py
@@ -20,7 +20,7 @@ from scripts.test import shared
 from scripts.test import support
 
 tests = shared.get_tests(shared.options.binaryen_test)
-spec_tests = shared.get_tests(shared.get_test_dir('spec'), ['.wast'])
+spec_tests = shared.options.spec_tests
 spec_tests = [t for t in spec_tests if '.fail' not in t]
 wasm2js_tests = shared.get_tests(shared.get_test_dir('wasm2js'), ['.wast'])
 assert_tests = ['wasm2js.wast.asserts']
-- 
cgit v1.2.3