Enable more spec tests (#6669)

Re-triage all the disabled spec tests and re-enable many of them.
Improve the module splitting logic to correctly handle (by skipping)
quoted modules and their associated assertions.

4 files changed, 83 insertions, 89 deletions

diff --git a/check.py b/check.py
index 946e3a0cf..66493dd3c 100755
--- a/check.py
+++ b/check.py
@@ -221,38 +221,30 @@ def run_spec_tests():
 
         check_expected(actual, expected)
 
-        # skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature
-        if 'exports.wast' in base:  # FIXME
-            continue
-
         run_spec_test(wast)
 
         # check binary format. here we can verify execution of the final
         # result, no need for an output verification
-        # some wast files cannot be split:
-        #     * comments.wast: contains characters that are not valid utf-8,
-        #       so our string splitting code fails there
-
-        # FIXME Remove reference type tests from this list after nullref is
-        # implemented in V8
-        if base not in ['comments.wast', 'ref_null.wast', 'ref_is_null.wast', 'ref_func.wast', 'old_select.wast']:
-            split_num = 0
-            actual = ''
-            with open('spec.wast', 'w') as transformed_spec_file:
-                for module, asserts in support.split_wast(wast):
-                    print('        testing split module', split_num)
-                    split_num += 1
-                    support.write_wast('split.wast', module)
-                    run_opt_test('split.wast')    # also that our optimizer doesn't break on it
-                    result_wast_file = shared.binary_format_check('split.wast', verify_final_result=False)
-                    with open(result_wast_file) as f:
-                        result_wast = f.read()
-                        # add the asserts, and verify that the test still passes
-                        transformed_spec_file.write(result_wast + '\n' + '\n'.join(asserts))
-
-            # compare all the outputs to the expected output
-            actual = run_spec_test('spec.wast')
-            check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', base + '.log'))
+        split_num = 0
+        actual = ''
+        with open('spec.wast', 'w') as transformed_spec_file:
+            for module, asserts in support.split_wast(wast):
+                if not module:
+                    # Skip any initial assertions that don't have a module
+                    continue
+                print('        testing split module', split_num)
+                split_num += 1
+                support.write_wast('split.wast', module)
+                run_opt_test('split.wast')    # also that our optimizer doesn't break on it
+                result_wast_file = shared.binary_format_check('split.wast', verify_final_result=False)
+                with open(result_wast_file) as f:
+                    result_wast = f.read()
+                    # add the asserts, and verify that the test still passes
+                    transformed_spec_file.write(result_wast + '\n' + '\n'.join(asserts))
+
+        # compare all the outputs to the expected output
+        actual = run_spec_test('spec.wast')
+        check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', base + '.log'))
 
 
 def run_validator_tests():
diff --git a/scripts/test/shared.py b/scripts/test/shared.py
index 68c8b9d94..847dd9de5 100644
--- a/scripts/test/shared.py
+++ b/scripts/test/shared.py
@@ -399,57 +399,33 @@ os.chdir(options.out_dir)
 # delete the old file, make sure you rename the corresponding .wast.log file in
 # expected-output/ if any.
 SPEC_TESTS_TO_SKIP = [
-    # Stacky code / notation
-    'block.wast',
-    'call.wast',
-    'float_exprs.wast',
+    # Malformed module accepted
     'globals.wast',
-    'loop.wast',
-    'nop.wast',
-    'select.wast',
-    'stack.wast',
-    'unwind.wast',
-
-    # Binary module
-    'binary.wast',
     'binary-leb128.wast',
-    'custom.wast',
-
-    # Empty 'then' or 'else' in 'if'
-    'if.wast',
-    'local_set.wast',
-    'store.wast',
-
-    # No module in a file
-    'token.wast',
     'utf8-custom-section-id.wast',
     'utf8-import-field.wast',
     'utf8-import-module.wast',
     'utf8-invalid-encoding.wast',
+    'const.wast',
+    'address.wast',
+    'custom.wast',  # invalid section ID accepted as Custom, triggering UBSan
 
-    # 'register' command
+    # Unlinkable module accepted
     'linking.wast',
 
-    # Misc. unsupported constructs
-    'call_indirect.wast',  # Empty (param) and (result)
-    'const.wast',  # Unparenthesized expression
-    'data.wast',  # Various unsupported (data) notations
-    'elem.wast',  # Unsupported 'offset' syntax in (elem)
-    'exports.wast',  # Multiple inlined exports for a function
-    'func.wast',  # Forward named type reference
-    'skip-stack-guard-page.wast',  # Hexadecimal style (0x..) in memory offset
-
-    # Untriaged: We don't know the cause of the error yet
-    'address.wast',  # wasm2js 'assert_return' failure
-    'br_if.wast',  # Validation error
-    'float_literals.wast',  # 'assert_return' failure
-    'int_literals.wast',  # 'assert_return' failure
-    'local_tee.wast',  # Validation failure
-    'memory_grow.wast',  # 'assert_return' failure
-    'start.wast',  # Assertion failure
-    'type.wast',  # 'assertion_invalid' failure
-    'unreachable.wast',  # Validation failure
-    'unreached-invalid.wast'  # 'assert_invalid' failure
+    # Invalid module accepted
+    'func.wast',
+    'type.wast',
+    'unreached-invalid.wast',
+
+    # WAT parser error
+    'unwind.wast',
+
+    # WAST parser error
+    'binary.wast',
+
+    # Test invalid
+    'elem.wast',
 ]
 options.spec_tests = [t for t in options.spec_tests if os.path.basename(t) not
                       in SPEC_TESTS_TO_SKIP]
diff --git a/scripts/test/support.py b/scripts/test/support.py
index 3a6ed9bc2..43bd00fe8 100644
--- a/scripts/test/support.py
+++ b/scripts/test/support.py
@@ -14,6 +14,7 @@
 
 import filecmp
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -87,6 +88,9 @@ def untar(tarfile, outdir):
             shutil.rmtree(tmpdir)
 
 
+QUOTED = re.compile(r'\(module\s*(\$\S*)?\s+(quote|binary)')
+
+
 def split_wast(wastFile):
     # if it's a binary, leave it as is, we can't split it
     wast = None
@@ -124,6 +128,7 @@ def split_wast(wastFile):
         return j
 
     i = 0
+    ignoring_quoted = False
     while i >= 0:
         start = wast.find('(', i)
         if start >= 0 and wast[start + 1] == ';':
@@ -141,11 +146,17 @@ def split_wast(wastFile):
             break
         i = to_end(start + 1)
         chunk = wast[start:i]
+        if QUOTED.match(chunk):
+            # There may be assertions after this quoted module, but we aren't
+            # returning the module, so we need to skip the assertions as well.
+            ignoring_quoted = True
+            continue
         if chunk.startswith('(module'):
+            ignoring_quoted = False
             ret += [(chunk, [])]
         elif chunk.startswith('(assert_invalid'):
             continue
-        elif chunk.startswith(('(assert', '(invoke', '(register')):
+        elif chunk.startswith(('(assert', '(invoke', '(register')) and not ignoring_quoted:
             # ret may be empty if there are some asserts before the first
             # module. in that case these are asserts *without* a module, which
             # are valid (they may check something that doesn't refer to a module
diff --git a/test/spec/stack.wast b/test/spec/stack.wast
index a8c79b698..5da5043c8 100644
--- a/test/spec/stack.wast
+++ b/test/spec/stack.wast
@@ -34,17 +34,15 @@
         (i64.eq)
         (if
           (then (br $done))
-          (then
-            (else
-              (local.get $i)
-              (local.get $res)
-              (i64.mul)
-              (local.set $res)
-              (local.get $i)
-              (i64.const 1)
-              (i64.sub)
-              (local.set $i)
-            )
+          (else
+            (local.get $i)
+            (local.get $res)
+            (i64.mul)
+            (local.set $res)
+            (local.get $i)
+            (i64.const 1)
+            (i64.sub)
+            (local.set $i)
           )
         )
         (br $loop)
@@ -93,13 +91,11 @@
         (i64.eq (local.get $i) (i64.const 0))
         (if
           (then (br $done))
-          (then
-            (else
-              (i64.mul (local.get $i) (local.get $res))
-              (local.set $res)
-              (i64.sub (local.get $i) (i64.const 1))
-              (local.set $i)
-            )
+          (else
+            (i64.mul (local.get $i) (local.get $res))
+            (local.set $res)
+            (i64.sub (local.get $i) (i64.const 1))
+            (local.set $i)
           )
         )
         (br $loop)
@@ -129,12 +125,31 @@
     end
     local.get $res
   )
+
+  (global $temp (mut i32) (i32.const 0))
+  (func $add_one_to_global (result i32)
+    (local i32)
+    (global.set $temp (i32.add (i32.const 1) (global.get $temp)))
+    (global.get $temp)
+  )
+  (func $add_one_to_global_and_drop
+    (drop (call $add_one_to_global))
+  )
+  (func (export "not-quite-a-tree") (result i32)
+    call $add_one_to_global
+    call $add_one_to_global
+    call $add_one_to_global_and_drop
+    i32.add
+  )
 )
 
 (assert_return (invoke "fac-expr" (i64.const 25)) (i64.const 7034535277573963776))
 (assert_return (invoke "fac-stack" (i64.const 25)) (i64.const 7034535277573963776))
 (assert_return (invoke "fac-mixed" (i64.const 25)) (i64.const 7034535277573963776))
 
+(assert_return (invoke "not-quite-a-tree") (i32.const 3))
+(assert_return (invoke "not-quite-a-tree") (i32.const 9))
+
 
 ;; Syntax of flat call_indirect