diff options
Diffstat (limited to 'test/run-tests.py')
| -rwxr-xr-x | test/run-tests.py | 167 |
1 files changed, 93 insertions, 74 deletions
diff --git a/test/run-tests.py b/test/run-tests.py index 63a5bb91..6d559e7d 100755 --- a/test/run-tests.py +++ b/test/run-tests.py @@ -31,19 +31,18 @@ import shlex import shutil import subprocess import sys -import tempfile import threading import time import find_exe -import findtests from utils import Error IS_WINDOWS = sys.platform == 'win32' -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -REPO_ROOT_DIR = os.path.dirname(SCRIPT_DIR) -ROUNDTRIP_PY = os.path.join(SCRIPT_DIR, 'run-roundtrip.py') +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT_DIR = os.path.dirname(TEST_DIR) +OUT_DIR = os.path.join(REPO_ROOT_DIR, 'out') +ROUNDTRIP_PY = os.path.join(TEST_DIR, 'run-roundtrip.py') DEFAULT_TIMEOUT = 10 # seconds SLOW_TIMEOUT_MULTIPLIER = 2 @@ -257,12 +256,10 @@ def RunCommandWithTimeout(command, cwd, timeout, console_out=False): class TestInfo(object): def __init__(self): - self.name = '' - self.generated_input_filename = '' self.filename = '' self.header = [] self.input_filename = '' - self.input_ = [] + self.input_ = '' self.expected_stdout = '' self.expected_stderr = '' self.tool = 'wast2wasm' @@ -276,8 +273,6 @@ class TestInfo(object): def CreateRoundtripInfo(self): result = TestInfo() - result.name = '%s (roundtrip)' % self.name - result.generated_input_filename = AppendBeforeExt(self.name, '-roundtrip') result.filename = self.filename result.header = self.header result.input_filename = self.input_filename @@ -294,6 +289,25 @@ class TestInfo(object): result.is_roundtrip = True return result + def GetName(self): + name = self.filename + if self.is_roundtrip: + name += ' (roundtrip)' + return name + + def GetGeneratedInputFilename(self): + path = OUT_DIR + if self.input_filename: + path = os.path.join(path, self.input_filename) + else: + path = os.path.join(path, self.filename) + + if self.is_roundtrip: + dirname = os.path.dirname(path) + basename = os.path.basename(path) + path = os.path.join(dirname, 'roundtrip', basename) + return path + def ShouldCreateRoundtrip(self): return self.tool in ROUNDTRIP_TOOLS @@ -302,7 +316,6 @@ class TestInfo(object): self.exe = value elif key == 'STDIN_FILE': self.input_filename = value - self.generated_input_filename = value elif key == 'FLAGS': self.flags += shlex.split(value) elif key == 'ERROR': @@ -325,11 +338,10 @@ class TestInfo(object): raise Error('Unknown directive: %s' % key) def Parse(self, filename): - self.name = filename - self.filename = os.path.join(SCRIPT_DIR, filename) - self.generated_input_filename = self.name + self.filename = filename - with open(self.filename) as f: + test_path = os.path.join(REPO_ROOT_DIR, filename) + with open(test_path) as f: seen_keys = set() state = 'header' empty = True @@ -411,32 +423,31 @@ class TestInfo(object): self.last_cmd = cmd return cmd - def CreateInputFile(self, temp_dir): - file_path = os.path.join(temp_dir, self.generated_input_filename) - file_dir = os.path.dirname(file_path) + def CreateInputFile(self): + gen_input_path = self.GetGeneratedInputFilename() + gen_input_dir = os.path.dirname(gen_input_path) try: - os.makedirs(file_dir) + os.makedirs(gen_input_dir) except OSError as e: - if not os.path.isdir(file_dir): + if not os.path.isdir(gen_input_dir): raise - file_ = open(file_path, 'wb') - # add an empty line for each header line so the line numbers match - if self.input_filename: - with open(self.input_filename, 'rb') as input_filename: - file_.write(input_filename.read()) - else: - file_.write(('\n' * self.header.count('\n')).encode('ascii')) - file_.write(self.input_.encode('ascii')) - file_.flush() - file_.close() - - # make filenames relative to temp_dir so the autogenerated name is not - # included in the error output. - return os.path.relpath(file_.name, temp_dir) + # Read/write as binary because spec tests may have invalid UTF-8 codes. + with open(gen_input_path, 'wb') as gen_input_file: + if self.input_filename: + input_path = os.path.join(REPO_ROOT_DIR, self.input_filename) + with open(input_path, 'rb') as input_file: + gen_input_file.write(input_file.read()) + else: + # add an empty line for each header line so the line numbers match + gen_input_file.write(('\n' * self.header.count('\n')).encode('ascii')) + gen_input_file.write(self.input_.encode('ascii')) + gen_input_file.flush() + return gen_input_file.name def Rebase(self, stdout, stderr): - with open(self.filename, 'wb') as f: + test_path = os.path.join(REPO_ROOT_DIR, self.filename) + with open(test_path, 'w', newline='') as f: f.write(self.header) f.write(self.input_) if stderr: @@ -483,7 +494,7 @@ class Status(object): def Passed(self, info, duration): self.passed += 1 if self.verbose: - sys.stderr.write('+ %s (%.3fs)\n' % (info.name, duration)) + sys.stderr.write('+ %s (%.3fs)\n' % (info.GetName(), duration)) else: self.Clear() self._PrintShortStatus(info) @@ -494,12 +505,12 @@ class Status(object): self.failed_tests.append(info) if not self.verbose: self.Clear() - sys.stderr.write('- %s\n%s\n' % (info.name, Indent(error_msg, 2))) + sys.stderr.write('- %s\n%s\n' % (info.GetName(), Indent(error_msg, 2))) def Skipped(self, info): self.skipped += 1 if self.verbose: - sys.stderr.write('. %s (skipped)\n' % info.name) + sys.stderr.write('. %s (skipped)\n' % info.GetName()) def UpdateTimer(self): self._PrintShortStatus(self.last_finished) @@ -510,7 +521,7 @@ class Status(object): def _PrintShortStatus(self, info): total_duration = time.time() - self.start_time - name = info.name if info else '' + name = info.GetName() if info else '' if (self.total - self.skipped): percent = 100 * (self.passed + self.failed) / (self.total - self.skipped) else: @@ -526,6 +537,17 @@ class Status(object): sys.stderr.write('%s\r' % (' ' * self.last_length)) +def FindTestFiles(ext, filter_pattern_re): + tests = [] + for root, dirs, files in os.walk(TEST_DIR): + for f in files: + path = os.path.join(root, f) + if os.path.splitext(f)[1] == ext: + tests.append(os.path.relpath(path, REPO_ROOT_DIR)) + tests.sort() + return [test for test in tests if re.match(filter_pattern_re, test)] + + def GetAllTestInfo(test_names, status): infos = [] for test_name in test_names: @@ -535,23 +557,29 @@ def GetAllTestInfo(test_names, status): infos.append(info) except Error as e: status.Failed(info, str(e)) - return infos + def RunTest(info, options, variables, verbose_level = 0): timeout = options.timeout if info.slow: timeout *= SLOW_TIMEOUT_MULTIPLIER + # Clone variables dict so it can be safely modified. + variables = dict(variables) + try: - rel_file_path = info.CreateInputFile(variables['out_dir']) - cmd = info.GetCommand(rel_file_path, variables, options.arg, verbose_level) - out = RunCommandWithTimeout(cmd, variables['out_dir'], timeout, - verbose_level > 0) - return out + cwd = REPO_ROOT_DIR + gen_input_path = info.CreateInputFile() + rel_gen_input_path = os.path.relpath(gen_input_path, cwd) + variables['out_dir'] = os.path.dirname(rel_gen_input_path) + cmd = info.GetCommand(rel_gen_input_path, variables, options.arg, + verbose_level) + return RunCommandWithTimeout(cmd, cwd, timeout, verbose_level > 0) except Exception as e: return e + def ThreadWorker(i, options, variables, inq, outq, should_run): try: while should_run.Get(): @@ -567,6 +595,7 @@ def ThreadWorker(i, options, variables, inq, outq, should_run): except KeyboardInterrupt: pass + def HandleTestResult(status, info, result, rebase=False): try: if isinstance(result, Exception): @@ -601,6 +630,7 @@ def HandleTestResult(status, info, result, rebase=False): except Exception as e: status.Failed(info, str(e)) + #Source : http://stackoverflow.com/questions/3041986/python-command-line-yes-no-input def YesNoPrompt(question, default='yes'): """Ask a yes/no question via raw_input() and return their answer. @@ -634,15 +664,12 @@ def YesNoPrompt(question, default='yes'): sys.stdout.write('Please respond with \'yes\' or \'no\' ' '(or \'y\' or \'n\').\n') -def WaitWorkersTerminate(workers, timeout=5): - for worker in workers: - if worker.is_alive(): - worker.join(timeout) -def RunMultiThreaded(infos_to_run, test_count, status, options, variables): +def RunMultiThreaded(infos_to_run, status, options, variables): should_stop_on_error = options.stop_interactive continued_errors = 0 num_threads = options.jobs + test_count = len(infos_to_run) all_threads = [] try: @@ -676,7 +703,10 @@ def RunMultiThreaded(infos_to_run, test_count, status, options, variables): continued_errors += 1 finally: should_run.Set(False) - WaitWorkersTerminate(all_threads) + for thread in all_threads: + if thread.is_alive(): + thread.join(timeout=5) + def RunSingleThreaded(infos_to_run, status, options, variables): continued_errors = 0 @@ -700,6 +730,7 @@ def RunSingleThreaded(infos_to_run, status, options, variables): RunTest(info, options, variables, verbose_level=1) continued_errors += 1 + def GetDefaultJobCount(): cpu_count = multiprocessing.cpu_count() if cpu_count <= 1: @@ -707,13 +738,12 @@ def GetDefaultJobCount(): else: return cpu_count // 2 + def main(args): parser = argparse.ArgumentParser() parser.add_argument('-a', '--arg', help='additional args to pass to executable', action='append') - parser.add_argument('-o', '--out-dir', metavar='PATH', - help='output directory for files.') parser.add_argument('--exe-dir', metavar='PATH', help='directory to search for all executables. ' 'This can be overridden by the other executable ' @@ -759,7 +789,7 @@ def main(args): else: pattern_re = '.*' - test_names = findtests.FindTestFiles(SCRIPT_DIR, '.txt', pattern_re) + test_names = FindTestFiles('.txt', pattern_re) if options.list: for test_name in test_names: @@ -770,7 +800,7 @@ def main(args): return 1 variables = {} - variables['test_dir'] = os.path.abspath(SCRIPT_DIR) + variables['test_dir'] = os.path.abspath(TEST_DIR) for exe_basename in find_exe.EXECUTABLES: attr_name = exe_basename.replace('-', '_') @@ -792,33 +822,21 @@ def main(args): continue infos_to_run.append(info) - if options.roundtrip: - if info.ShouldCreateRoundtrip(): - infos_to_run.append(info.CreateRoundtripInfo()) + if options.roundtrip and info.ShouldCreateRoundtrip(): + infos_to_run.append(info.CreateRoundtripInfo()) - test_count = len(infos_to_run) - status.Start(test_count) + if not os.path.exists(OUT_DIR): + os.makedirs(OUT_DIR) - if options.out_dir: - out_dir = options.out_dir - out_dir_is_temp = False - if not os.path.exists(out_dir): - os.makedirs(out_dir) - else: - out_dir = tempfile.mkdtemp(prefix='wabt-') - out_dir_is_temp = True - variables['out_dir'] = os.path.abspath(out_dir) + status.Start(len(infos_to_run)) try: if options.jobs > 1: - RunMultiThreaded(infos_to_run, test_count, status, options, variables) + RunMultiThreaded(infos_to_run, status, options, variables) else: RunSingleThreaded(infos_to_run, status, options, variables) except KeyboardInterrupt: print('\nInterrupted testing\n') - finally: - if out_dir_is_temp: - shutil.rmtree(out_dir) status.Clear() @@ -827,7 +845,8 @@ def main(args): if status.failed: sys.stderr.write('**** FAILED %s\n' % ('*' * (80 - 14))) for info in status.failed_tests: - sys.stderr.write('- %s\n %s\n' % (info.name, ' '.join(info.last_cmd))) + sys.stderr.write('- %s\n %s\n' % (info.GetName(), + ' '.join(info.last_cmd))) ret = 1 status.Print() |