summaryrefslogtreecommitdiff
path: root/scripts/test
diff options
context:
space:
mode:
authorNathan Froyd <froydnj@gmail.com>2018-03-19 17:02:40 -0400
committerAlon Zakai <alonzakai@gmail.com>2018-03-19 14:02:40 -0700
commit267e4b55f8c2f598ea5ea44d5df88d83bfb3ebac (patch)
tree8c2aaad800cbdaf9f92f28ec9c7d480e906342b0 /scripts/test
parent9c9b78db2e6ce178cc0c546d59aacaa4c72c001b (diff)
downloadbinaryen-267e4b55f8c2f598ea5ea44d5df88d83bfb3ebac.tar.gz
binaryen-267e4b55f8c2f598ea5ea44d5df88d83bfb3ebac.tar.bz2
binaryen-267e4b55f8c2f598ea5ea44d5df88d83bfb3ebac.zip
include the filename in erroneous diff output where possible (#1452)
Failing test cases often start out with: ``` incorrect output, diff: --- expected +++ actual ``` which makes it difficult to figure out where the expected output might live. That information can be derived from examining the tests, of course, but it'd be much nicer if it were provided in the diff to see straightaway. We do this by introducing a new check, one which takes a filename of expected output, which then enables us to display the failing file, e.g.: ``` incorrect output, diff: --- /home/froydnj/src/binaryen.git/test/passes/code-folding.txt +++ actual ``` which is arguably nicer. Having this new check also enables reducing some boilerplate `open(...).read()` calls in various places. There are still a few places using `fail_if_not_identical`, usually because `.strip()` is used on the expected output.
Diffstat (limited to 'scripts/test')
-rwxr-xr-x[-rw-r--r--]scripts/test/asm2wasm.py11
-rwxr-xr-xscripts/test/lld.py8
-rwxr-xr-xscripts/test/s2wasm.py8
-rw-r--r--scripts/test/shared.py17
-rwxr-xr-xscripts/test/wasm2asm.py12
5 files changed, 25 insertions, 31 deletions
diff --git a/scripts/test/asm2wasm.py b/scripts/test/asm2wasm.py
index abcd41bb4..3fb94fd1d 100644..100755
--- a/scripts/test/asm2wasm.py
+++ b/scripts/test/asm2wasm.py
@@ -20,7 +20,7 @@ import subprocess
from support import run_command
from shared import (
ASM2WASM, WASM_OPT, binary_format_check, delete_from_orbit,
- fail, fail_with_error, fail_if_not_identical, options, tests
+ fail_with_error, options, tests, fail_if_not_identical_to_file
)
@@ -69,9 +69,7 @@ def test_asm2wasm():
# verify output
if not os.path.exists(wasm):
fail_with_error('output .wast file %s does not exist' % wasm)
- expected = open(wasm, 'rb').read()
- if actual != expected:
- fail(actual, expected)
+ fail_if_not_identical_to_file(actual, wasm)
binary_format_check(wasm, verify_final_result=False)
@@ -128,9 +126,8 @@ def test_asm2wasm():
run_command(cmd)
if not os.path.isfile(jsmap):
fail_with_error('Debug info map not created: %s' % jsmap)
- with open(wasm + '.map', 'rb') as expected:
- with open(jsmap, 'rb') as actual:
- fail_if_not_identical(actual.read(), expected.read())
+ with open(jsmap, 'rb') as actual:
+ fail_if_not_identical_to_file(actual.read(), wasm + '.map')
with open('a.wasm', 'rb') as binary:
url_section_name = bytearray([16]) + bytearray('sourceMappingURL')
url = 'http://example.org/' + jsmap
diff --git a/scripts/test/lld.py b/scripts/test/lld.py
index f6aa91e6f..01ed51a4a 100755
--- a/scripts/test/lld.py
+++ b/scripts/test/lld.py
@@ -17,8 +17,8 @@
import os
from support import run_command
from shared import (
- fail, fail_with_error, files_with_pattern, options,
- WASM_EMSCRIPTEN_FINALIZE
+ fail_with_error, files_with_pattern, options,
+ WASM_EMSCRIPTEN_FINALIZE, fail_if_not_identical_to_file
)
@@ -44,9 +44,7 @@ def test_wasm_emscripten_finalize():
if not os.path.exists(expected_file):
print actual
fail_with_error('output ' + expected_file + ' does not exist')
- expected = open(expected_file, 'rb').read()
- if actual != expected:
- fail(actual, expected)
+ fail_if_not_identical_to_file(actual, expected_file)
if __name__ == '__main__':
diff --git a/scripts/test/s2wasm.py b/scripts/test/s2wasm.py
index a3345f87b..0e0bec621 100755
--- a/scripts/test/s2wasm.py
+++ b/scripts/test/s2wasm.py
@@ -17,8 +17,8 @@
import os
from support import run_command
from shared import (
- fail, fail_with_error, fail_if_not_contained,
- options, S2WASM, WASM_SHELL
+ fail_with_error, fail_if_not_contained,
+ options, S2WASM, WASM_SHELL, fail_if_not_identical_to_file
)
@@ -64,9 +64,7 @@ def test_s2wasm():
if not expected_exists:
print actual
fail_with_error('output ' + expected_file + ' does not exist')
- expected = open(expected_file, 'rb').read()
- if actual != expected:
- fail(actual, expected)
+ fail_if_not_identical_to_file(actual, expected_file)
# verify with options
cmd = S2WASM + [full, '--global-base=1024'] + stack_alloc
diff --git a/scripts/test/shared.py b/scripts/test/shared.py
index 23f10ca2c..9921d0ada 100644
--- a/scripts/test/shared.py
+++ b/scripts/test/shared.py
@@ -344,17 +344,17 @@ def fail_with_error(msg):
raise
-def fail(actual, expected):
+def fail(actual, expected, fromfile='expected'):
diff_lines = difflib.unified_diff(
expected.split('\n'), actual.split('\n'),
- fromfile='expected', tofile='actual')
+ fromfile=fromfile, tofile='actual')
diff_str = ''.join([a.rstrip() + '\n' for a in diff_lines])[:]
fail_with_error("incorrect output, diff:\n\n%s" % diff_str)
-def fail_if_not_identical(actual, expected):
+def fail_if_not_identical(actual, expected, fromfile='expected'):
if expected != actual:
- fail(actual, expected)
+ fail(actual, expected, fromfile=fromfile)
def fail_if_not_contained(actual, expected):
@@ -362,6 +362,11 @@ def fail_if_not_contained(actual, expected):
fail(actual, expected)
+def fail_if_not_identical_to_file(actual, expected_file):
+ with open(expected_file, 'rb') as f:
+ fail_if_not_identical(actual, f.read(), fromfile=expected_file)
+
+
if len(requested) == 0:
tests = sorted(os.listdir(os.path.join(options.binaryen_test)))
else:
@@ -401,10 +406,8 @@ def binary_format_check(wast, verify_final_result=True, wasm_as_args=['-g'],
subprocess.check_call(cmd, stdout=subprocess.PIPE)
if verify_final_result:
- expected = open(wast + binary_suffix).read()
actual = open('ab.wast').read()
- if actual != expected:
- fail(actual, expected)
+ fail_if_not_identical_to_file(actual, wast + binary_suffix)
return 'ab.wast'
diff --git a/scripts/test/wasm2asm.py b/scripts/test/wasm2asm.py
index 6e6b32c81..81cb9b5c5 100755
--- a/scripts/test/wasm2asm.py
+++ b/scripts/test/wasm2asm.py
@@ -4,7 +4,8 @@ import os
from support import run_command
from shared import (
- WASM2ASM, MOZJS, NODEJS, fail_if_not_identical, options, tests
+ WASM2ASM, MOZJS, NODEJS, fail_if_not_identical, options, tests,
+ fail_if_not_identical_to_file
)
# tests with i64s, invokes, etc.
@@ -33,8 +34,7 @@ def test_wasm2asm_output():
cmd = WASM2ASM + [os.path.join(options.binaryen_test, wasm)]
out = run_command(cmd)
- expected = open(expected_file).read()
- fail_if_not_identical(out, expected)
+ fail_if_not_identical_to_file(out, expected_file)
if not NODEJS and not MOZJS:
print 'No JS interpreters. Skipping spec tests.'
@@ -79,13 +79,11 @@ def test_asserts_output():
wasm = os.path.join(options.binaryen_test, wasm)
cmd = WASM2ASM + [wasm, '--allow-asserts']
out = run_command(cmd)
- expected = open(asserts_expected_file).read()
- fail_if_not_identical(out, expected)
+ fail_if_not_identical_to_file(out, asserts_expected_file)
cmd += ['--pedantic']
out = run_command(cmd)
- expected = open(traps_expected_file).read()
- fail_if_not_identical(out, expected)
+ fail_if_not_identical_to_file(out, traps_expected_file)
def test_wasm2asm():