summaryrefslogtreecommitdiff
path: root/test/run-tests.py
diff options
context:
space:
mode:
Diffstat (limited to 'test/run-tests.py')
-rwxr-xr-xtest/run-tests.py303
1 files changed, 194 insertions, 109 deletions
diff --git a/test/run-tests.py b/test/run-tests.py
index 4314b0c9..984139ac 100755
--- a/test/run-tests.py
+++ b/test/run-tests.py
@@ -43,19 +43,22 @@ SLOW_TIMEOUT_MULTIPLIER = 2
TOOLS = {
'wat2wasm': {
'EXE': '%(wat2wasm)s',
- 'FLAGS': [ '-o', '%(out_dir)s/out.wasm'],
+ 'FLAGS': ['%(in_file)s', '-o', '%(out_dir)s/out.wasm'],
'VERBOSE-FLAGS': ['-v']
},
'wast2json': {
'EXE': '%(wast2json)s',
+ 'FLAGS': ['%(in_file)s'],
'VERBOSE-FLAGS': ['-v']
},
'wat-desugar': {
- 'EXE': '%(wat-desugar)s'
+ 'EXE': '%(wat-desugar)s',
+ 'FLAGS': ['%(in_file)s']
},
'run-objdump': {
'EXE': 'test/run-objdump.py',
'FLAGS': [
+ '%(in_file)s',
'--bindir=%(bindir)s',
'--no-error-cmdline',
'-o', '%(out_dir)s'
@@ -64,12 +67,13 @@ TOOLS = {
},
'run-wasm-link': {
'EXE': 'test/run-wasm-link.py',
- 'FLAGS': ['--bindir=%(bindir)s', '-o', '%(out_dir)s'],
+ 'FLAGS': ['%(in_file)s', '--bindir=%(bindir)s', '-o', '%(out_dir)s'],
'VERBOSE-FLAGS': ['-v']
},
'run-roundtrip': {
'EXE': 'test/run-roundtrip.py',
'FLAGS': [
+ '%(in_file)s',
'-v',
'--bindir=%(bindir)s',
'--no-error-cmdline',
@@ -81,6 +85,7 @@ TOOLS = {
'run-interp': {
'EXE': 'test/run-interp.py',
'FLAGS': [
+ '%(in_file)s',
'--bindir=%(bindir)s',
'--run-all-exports',
'--no-error-cmdline',
@@ -92,6 +97,7 @@ TOOLS = {
'run-interp-spec': {
'EXE': 'test/run-interp.py',
'FLAGS': [
+ '%(in_file)s',
'--bindir=%(bindir)s',
'--spec',
'--no-error-cmdline',
@@ -103,6 +109,7 @@ TOOLS = {
'run-gen-wasm': {
'EXE': 'test/run-gen-wasm.py',
'FLAGS': [
+ '%(in_file)s',
'--bindir=%(bindir)s',
'--no-error-cmdline',
'-o',
@@ -113,6 +120,7 @@ TOOLS = {
'run-gen-wasm-interp': {
'EXE': 'test/run-gen-wasm-interp.py',
'FLAGS': [
+ '%(in_file)s',
'--bindir=%(bindir)s',
'--run-all-exports',
'--no-error-cmdline',
@@ -124,6 +132,7 @@ TOOLS = {
'run-opcodecnt': {
'EXE': 'test/run-opcodecnt.py',
'FLAGS': [
+ '%(in_file)s',
'--bindir=%(bindir)s',
'--no-error-cmdline',
'-o',
@@ -134,6 +143,7 @@ TOOLS = {
'run-gen-spec-js': {
'EXE': 'test/run-gen-spec-js.py',
'FLAGS': [
+ '%(in_file)s',
'--bindir=%(bindir)s',
'--no-error-cmdline',
'-o',
@@ -144,6 +154,7 @@ TOOLS = {
'run-spec-wasm2c': {
'EXE': 'test/run-spec-wasm2c.py',
'FLAGS': [
+ '%(in_file)s',
'--bindir=%(bindir)s',
'--no-error-cmdline',
'-o',
@@ -160,6 +171,10 @@ if IS_WINDOWS:
ROUNDTRIP_TOOLS = ('wat2wasm',)
+class NoRoundtripError(Error):
+ pass
+
+
def Indent(s, spaces):
return ''.join(' ' * spaces + l for l in s.splitlines(1))
@@ -172,11 +187,6 @@ def DiffLines(expected, actual):
tofile='actual', lineterm=''))
-def AppendBeforeExt(file_path, suffix):
- file_path_noext, ext = os.path.splitext(file_path)
- return file_path_noext + suffix + ext
-
-
class Cell(object):
def __init__(self, value):
@@ -189,52 +199,131 @@ class Cell(object):
return self.value[0]
-def RunCommandWithTimeout(command, cwd, timeout, console_out=False, env=None):
- process = None
- is_timeout = Cell(False)
+def SplitArgs(value):
+ if isinstance(value, list):
+ return value
+ return shlex.split(value)
- def KillProcess(timeout=True):
- if process:
- try:
- if IS_WINDOWS:
- # http://stackoverflow.com/a/10830753: deleting child processes in
- # Windows
- subprocess.call(['taskkill', '/F', '/T', '/PID', str(process.pid)])
- else:
- os.killpg(os.getpgid(process.pid), 15)
- except OSError:
- pass
- is_timeout.Set(timeout)
- try:
- start_time = time.time()
- kwargs = {}
- if not IS_WINDOWS:
- kwargs['preexec_fn'] = os.setsid
-
- # http://stackoverflow.com/a/10012262: subprocess with a timeout
- # http://stackoverflow.com/a/22582602: kill subprocess and children
- process = subprocess.Popen(command, cwd=cwd, env=env,
- stdout=None if console_out else subprocess.PIPE,
- stderr=None if console_out else subprocess.PIPE,
- universal_newlines=True, **kwargs)
- timer = threading.Timer(timeout, KillProcess)
+class CommandTemplate(object):
+
+ def __init__(self, exe):
+ self.exe = exe
+ self.flags = []
+ self.verbose_flags = []
+ self.last_cmd = None
+
+ def AppendFlags(self, flags):
+ self.flags += SplitArgs(flags)
+
+ def AppendVerboseFlags(self, flags_list):
+ for level, level_flags in enumerate(flags_list):
+ while level >= len(self.verbose_flags):
+ self.verbose_flags.append([])
+ self.verbose_flags[level] += SplitArgs(level_flags)
+
+ def _GetExecutable(self):
+ if os.path.splitext(self.exe)[1] == '.py':
+ return [sys.executable, os.path.join(REPO_ROOT_DIR, self.exe)]
+ else:
+ return [self.exe]
+
+ def _Format(self, cmd, variables):
+ return [arg % variables for arg in cmd]
+
+ def GetCommand(self, variables, extra_args=None, verbose_level=0):
+ args = self._GetExecutable()
+ vl = 0
+ while vl < verbose_level and vl < len(self.verbose_flags):
+ args += self.verbose_flags[vl]
+ vl += 1
+ if extra_args:
+ args += extra_args
+ args += self.flags
+ args = self._Format(args, variables)
+ self.last_cmd = args
+ return Command(args)
+
+
+class Command(object):
+
+ def __init__(self, args):
+ self.args = args
+
+ def Run(self, cwd, timeout, console_out=False, env=None):
+ process = None
+ is_timeout = Cell(False)
+
+ def KillProcess(timeout=True):
+ if process:
+ try:
+ if IS_WINDOWS:
+ # http://stackoverflow.com/a/10830753: deleting child processes in
+ # Windows
+ subprocess.call(['taskkill', '/F', '/T', '/PID', str(process.pid)])
+ else:
+ os.killpg(os.getpgid(process.pid), 15)
+ except OSError:
+ pass
+ is_timeout.Set(timeout)
+
try:
- timer.start()
- stdout, stderr = process.communicate()
+ start_time = time.time()
+ kwargs = {}
+ if not IS_WINDOWS:
+ kwargs['preexec_fn'] = os.setsid
+
+ # http://stackoverflow.com/a/10012262: subprocess with a timeout
+ # http://stackoverflow.com/a/22582602: kill subprocess and children
+ process = subprocess.Popen(self.args, cwd=cwd, env=env,
+ stdout=None if console_out else subprocess.PIPE,
+ stderr=None if console_out else subprocess.PIPE,
+ universal_newlines=True, **kwargs)
+ timer = threading.Timer(timeout, KillProcess)
+ try:
+ timer.start()
+ stdout, stderr = process.communicate()
+ finally:
+ returncode = process.returncode
+ process = None
+ timer.cancel()
+ if is_timeout.Get():
+ raise Error('TIMEOUT\nSTDOUT:\n%s\nSTDERR:\n%s\n' % (stdout, stderr))
+ duration = time.time() - start_time
+ except OSError as e:
+ raise Error(str(e))
finally:
- returncode = process.returncode
- process = None
- timer.cancel()
- if is_timeout.Get():
- raise Error('TIMEOUT\nSTDOUT:\n%s\nSTDERR:\n%s\n' % (stdout, stderr))
- duration = time.time() - start_time
- except OSError as e:
- raise Error(str(e))
- finally:
- KillProcess(False)
+ KillProcess(False)
+
+ return RunResult(stdout, stderr, returncode, duration)
- return stdout, stderr, returncode, duration
+ def __str__(self):
+ return ' '.join(self.args)
+
+
+class RunResult(object):
+
+ def __init__(self, stdout = '', stderr = '', returncode = 0, duration = 0):
+ self.stdout = stdout
+ self.stderr = stderr
+ self.returncode = returncode
+ self.duration = duration
+
+ def Failed(self):
+ return self.returncode != 0
+
+ def Append(self, other):
+ assert isinstance(other, RunResult)
+ self.stdout += other.stdout
+ self.stderr += other.stderr
+ self.duration += other.duration
+
+ assert(self.returncode == 0)
+ self.returncode = other.returncode
+
+ def __repr__(self):
+ return 'RunResult(%s, %s, %s, %s)' % (self.stdout, self.stderr,
+ self.returncode, self.duration)
class TestInfo(object):
@@ -247,8 +336,7 @@ class TestInfo(object):
self.expected_stdout = ''
self.expected_stderr = ''
self.tool = None
- self.exe = None
- self.flags = []
+ self.cmds = []
self.env = {}
self.last_cmd = ''
self.expected_error = 0
@@ -257,6 +345,12 @@ class TestInfo(object):
self.is_roundtrip = False
def CreateRoundtripInfo(self, fold_exprs):
+ if self.tool not in ROUNDTRIP_TOOLS:
+ raise NoRoundtripError()
+
+ if len(self.cmds) != 1:
+ raise NoRoundtripError()
+
result = TestInfo()
result.SetTool('run-roundtrip')
result.filename = self.filename
@@ -265,11 +359,15 @@ class TestInfo(object):
result.input_ = self.input_
result.expected_stdout = ''
result.expected_stderr = ''
+
# TODO(binji): It's kind of cheesy to keep the enable flag based on prefix.
# Maybe it would be nicer to add a new directive ENABLE instead.
- result.flags = [flag for flag in self.flags if flag.startswith('--enable')]
+ old_cmd = self.cmds[0]
+ new_cmd = result.cmds[0]
+ new_cmd.AppendFlags([f for f in old_cmd.flags if f.startswith('--enable')])
if fold_exprs:
- result.flags.append('--fold-exprs')
+ new_cmd.AppendFlags('--fold-exprs')
+
result.env = self.env
result.expected_error = 0
result.slow = self.slow
@@ -311,9 +409,6 @@ class TestInfo(object):
return path
- def ShouldCreateRoundtrip(self):
- return self.tool in ROUNDTRIP_TOOLS
-
def SetTool(self, tool):
if tool not in TOOLS:
raise Error('Unknown tool: %s' % tool)
@@ -321,15 +416,20 @@ class TestInfo(object):
for tool_key, tool_value in TOOLS[tool].items():
self.ParseDirective(tool_key, tool_value)
+ def GetLastCommand(self):
+ if not self.cmds:
+ self.SetTool('wat2wasm')
+ return self.cmds[-1]
+
def ParseDirective(self, key, value):
if key == 'EXE':
- self.exe = value
+ self.cmds.append(CommandTemplate(value))
elif key == 'STDIN_FILE':
self.input_filename = value
elif key == 'FLAGS':
if not isinstance(value, list):
value = shlex.split(value)
- self.flags += value
+ self.GetLastCommand().AppendFlags(value)
elif key == 'ERROR':
self.expected_error = int(value)
elif key == 'SLOW':
@@ -337,7 +437,7 @@ class TestInfo(object):
elif key == 'SKIP':
self.skip = True
elif key == 'VERBOSE-FLAGS':
- self.verbose_flags = [shlex.split(level) for level in value]
+ self.GetLastCommand().AppendVerboseFlags(value)
elif key in ['TODO', 'NOTE']:
pass
elif key == 'TOOL':
@@ -353,7 +453,6 @@ class TestInfo(object):
test_path = os.path.join(REPO_ROOT_DIR, filename)
with open(test_path) as f:
- seen_keys = set()
state = 'header'
empty = True
header_lines = []
@@ -379,9 +478,6 @@ class TestInfo(object):
key, value = directive.split(':', 1)
key = key.strip()
value = value.strip()
- if key in seen_keys:
- raise Error('%s already set' % key)
- seen_keys.add(key)
self.ParseDirective(key, value)
elif state in ('stdout', 'stderr'):
if not re.match(r'%s ;;\)$' % state.upper(), directive):
@@ -413,32 +509,9 @@ class TestInfo(object):
# If the test didn't specify a executable (either via EXE or indirectly
# via TOOL, then use the default tool)
- if not self.exe:
+ if not self.cmds:
self.SetTool('wat2wasm')
- def GetExecutable(self):
- if os.path.splitext(self.exe)[1] == '.py':
- return [sys.executable, os.path.join(REPO_ROOT_DIR, self.exe)]
- else:
- return [self.exe]
-
- def FormatCommand(self, cmd, variables):
- return [arg % variables for arg in cmd]
-
- def GetCommand(self, filename, variables, extra_args=None, verbose_level=0):
- cmd = self.GetExecutable()
- vl = 0
- while vl < verbose_level and vl < len(self.verbose_flags):
- cmd += self.verbose_flags[vl]
- vl += 1
- if extra_args:
- cmd += extra_args
- cmd += self.flags
- cmd += [filename.replace(os.path.sep, '/')]
- cmd = self.FormatCommand(cmd, variables)
- self.last_cmd = cmd
- return cmd
-
def CreateInputFile(self):
gen_input_path = self.GetGeneratedInputFilename()
gen_input_dir = os.path.dirname(gen_input_path)
@@ -587,7 +660,9 @@ def RunTest(info, options, variables, verbose_level=0):
env = dict(os.environ)
env.update(info.env)
gen_input_path = info.CreateInputFile()
- rel_gen_input_path = os.path.relpath(gen_input_path, cwd)
+ rel_gen_input_path = (
+ os.path.relpath(gen_input_path, cwd).replace(os.path.sep, '/'))
+ variables['in_file'] = rel_gen_input_path
# Each test runs with a unique output directory which is removed before
# we run the test.
@@ -597,15 +672,23 @@ def RunTest(info, options, variables, verbose_level=0):
os.makedirs(out_dir)
variables['out_dir'] = out_dir
- cmd = info.GetCommand(rel_gen_input_path, variables, options.arg,
- verbose_level)
- if options.print_cmd:
- print(' '.join(cmd))
+ final_result = RunResult()
- try:
- return RunCommandWithTimeout(cmd, cwd, timeout, verbose_level > 0, env)
- except (Error, KeyboardInterrupt) as e:
- return e
+ for cmd_template in info.cmds:
+ cmd = cmd_template.GetCommand(variables, options.arg, verbose_level)
+ if options.print_cmd:
+ print(cmd)
+
+ try:
+ result = cmd.Run(cwd, timeout, verbose_level > 0, env)
+ except (Error, KeyboardInterrupt) as e:
+ return e
+
+ final_result.Append(result)
+ if final_result.Failed():
+ break
+
+ return final_result
def HandleTestResult(status, info, result, rebase=False):
@@ -613,32 +696,31 @@ def HandleTestResult(status, info, result, rebase=False):
if isinstance(result, (Error, KeyboardInterrupt)):
raise result
- stdout, stderr, returncode, duration = result
if info.is_roundtrip:
- if returncode == 0:
- status.Passed(info, duration)
- elif returncode == 2:
+ if result.returncode == 0:
+ status.Passed(info, result.duration)
+ elif result.returncode == 2:
# run-roundtrip.py returns 2 if the file couldn't be parsed.
# it's likely a "bad-*" file.
status.Skipped(info)
else:
raise Error(stderr)
else:
- if returncode != info.expected_error:
+ if result.returncode != info.expected_error:
# This test has already failed, but diff it anyway.
msg = 'expected error code %d, got %d.' % (info.expected_error,
- returncode)
+ result.returncode)
try:
- info.Diff(stdout, stderr)
+ info.Diff(result.stdout, result.stderr)
except Error as e:
msg += '\n' + str(e)
raise Error(msg)
else:
if rebase:
- info.Rebase(stdout, stderr)
+ info.Rebase(result.stdout, result.stderr)
else:
- info.Diff(stdout, stderr)
- status.Passed(info, duration)
+ info.Diff(result.stdout, result.stderr)
+ status.Passed(info, result.duration)
except Error as e:
status.Failed(info, str(e))
@@ -802,9 +884,12 @@ def main(args):
continue
infos_to_run.append(info)
- if options.roundtrip and info.ShouldCreateRoundtrip():
- infos_to_run.append(info.CreateRoundtripInfo(fold_exprs=False))
- infos_to_run.append(info.CreateRoundtripInfo(fold_exprs=True))
+ if options.roundtrip:
+ for fold_exprs in False, True:
+ try:
+ infos_to_run.append(info.CreateRoundtripInfo(fold_exprs=fold_exprs))
+ except NoRoundtripError:
+ pass
if not os.path.exists(OUT_DIR):
os.makedirs(OUT_DIR)