path: root/check.py

#!/usr/bin/env python2

# Copyright 2015 WebAssembly Community Group participants
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os, shutil, sys, subprocess, difflib, json, time, urllib2, argparse

import scripts.storage
from scripts.support import run_command, split_wast

usage_str = "usage: 'python check.py [options]'\n\n Runs the Binaryen test suite."
parser = argparse.ArgumentParser(description=usage_str)
parser.add_argument('--torture', dest='torture', action='store_true', default=True, help='Chooses whether to run the torture testcases. Default: true.')
parser.add_argument('--no-torture', dest='torture', action='store_false', help='Disables running the torture testcases.')
parser.add_argument('--only-prepare', dest='only_prepare', action='store_true', default=False, help='If enabled, only fetches the waterfall build. Default: false.')
parser.add_argument('--only_prepare', dest='only_prepare', action='store_true', default=False, help='If enabled, only fetches the waterfall build. Default: false.') # Backwards compatibility
parser.add_argument('--test-waterfall', dest='test_waterfall', action='store_true', default=True, help='If enabled, fetches and tests the LLVM waterfall builds. Default: true.')
parser.add_argument('--no-test-waterfall', dest='test_waterfall', action='store_false', help='Disables downloading and testing of the LLVM waterfall builds.')
parser.add_argument('--abort-on-first-failure', dest='abort_on_first_failure', action='store_true', default=True, help='Specifies whether to halt test suite execution on first test error. Default: true.')
parser.add_argument('--no-abort-on-first-failure', dest='abort_on_first_failure', action='store_false', help='If set, the whole test suite will run to completion independent of earlier errors.')
parser.add_argument('--run-gcc-tests', dest='run_gcc_tests', action='store_true', default=True, help='Chooses whether to run the tests that require building with native GCC. Default: true.')
parser.add_argument('--no-run-gcc-tests', dest='run_gcc_tests', action='store_false', help='If set, disables the native GCC tests.')

parser.add_argument('--interpreter', dest='interpreter', default='', help='Specifies the wasm interpreter executable to run tests on.')
parser.add_argument('--binaryen-bin', dest='binaryen_bin', default='', help='Specifies a path to where the built Binaryen executables reside at. Default: bin/ of current directory (i.e. assume an in-tree build). If not specified, the environment variable BINARYEN_ROOT= can also be used to adjust this.')
parser.add_argument('--binaryen-root', dest='binaryen_root', default='', help='Specifies a path to the root of the Binaryen repository tree. Default: the directory where this file check.py resides.')
parser.add_argument('--valgrind', dest='valgrind', default='', help='Specifies a path to Valgrind tool, which will be used to validate execution if specified. (Pass --valgrind=valgrind to search in PATH)')
parser.add_argument('--valgrind-full-leak-check', dest='valgrind_full_leak_check', action='store_true', default=False, help='If specified, all unfreed (but still referenced) pointers at the end of execution are considered memory leaks. Default: disabled.')

parser.add_argument('positional_args', metavar='tests', nargs=argparse.REMAINDER, help='Names specific tests to run.')
options = parser.parse_args()
requested = options.positional_args

if options.interpreter:
  print '[ using wasm interpreter at "%s" ]' % options.interpreter
  assert os.path.exists(options.interpreter), 'interpreter not found'

num_failures = 0
warnings = []

def warn(text):
  global warnings
  warnings.append(text)
  print 'warning:', text

# setup

# Locate Binaryen build artifacts directory (bin/ by default)
if not options.binaryen_bin:
  if os.environ.get('BINARYEN_ROOT'):
    if os.path.isdir(os.path.join(os.environ.get('BINARYEN_ROOT'), 'bin')): options.binaryen_bin = os.path.join(os.environ.get('BINARYEN_ROOT'), 'bin')
    else: options.binaryen_bin = os.environ.get('BINARYEN_ROOT')
  else:
    options.binaryen_bin = 'bin'

if not os.path.isfile(os.path.join(options.binaryen_bin, 'wasm-dis')) and not os.path.isfile(os.path.join(options.binaryen_bin, 'wasm-dis.exe')):
  warn('Binaryen not found (or has not been successfully built to bin/ ?')

# Locate Binaryen source directory if not specified.
if not options.binaryen_root:
  options.binaryen_root = os.path.dirname(os.path.abspath(__file__))

options.binaryen_test = os.path.join(options.binaryen_root, 'test')

# Finds the given executable 'program' in PATH. Operates like the Unix tool 'which'.
def which(program):
  def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
  fpath, fname = os.path.split(program)
  if fpath:
    if is_exe(program): return program
  else:
    for path in os.environ["PATH"].split(os.pathsep):
      path = path.strip('"')
      exe_file = os.path.join(path, program)
      if is_exe(exe_file): return exe_file
      if not '.' in fname:
        if is_exe(exe_file + '.exe'): return exe_file + '.exe'
        if is_exe(exe_file + '.cmd'): return exe_file + '.cmd'
        if is_exe(exe_file + '.bat'): return exe_file + '.bat'

WATERFALL_BUILD_DIR = os.path.join(options.binaryen_test, 'wasm-install')
BIN_DIR = os.path.abspath(os.path.join(WATERFALL_BUILD_DIR, 'wasm-install', 'bin'))

NATIVECC = os.environ.get('CC') or which('mingw32-gcc') or which('gcc') or which('clang')
NATIVEXX = os.environ.get('CXX') or which('mingw32-g++') or which('g++') or which('clang++')
NODEJS = which('nodejs') or which('node')
MOZJS = which('mozjs')
EMCC = which('emcc')

WASM_OPT = [os.path.join(options.binaryen_bin, 'wasm-opt')]
WASM_AS = [os.path.join(options.binaryen_bin, 'wasm-as')]
WASM_DIS = [os.path.join(options.binaryen_bin, 'wasm-dis')]
ASM2WASM = [os.path.join(options.binaryen_bin, 'asm2wasm')]
WASM_SHELL = [os.path.join(options.binaryen_bin, 'wasm-shell')]
S2WASM = [os.path.join(options.binaryen_bin, 's2wasm')]

S2WASM_EXE = S2WASM[0]
WASM_SHELL_EXE = WASM_SHELL[0]

def wrap_with_valgrind(cmd):
  valgrind = [options.valgrind, '--quiet', '--error-exitcode=97'] # Exit code 97 is arbitrary, used to easily detect when an error occurs that is detected by Valgrind.
  if options.valgrind_full_leak_check:
    valgrind += ['--leak-check=full', '--show-leak-kinds=all']
  return valgrind + cmd

if options.valgrind:
  WASM_OPT = wrap_with_valgrind(WASM_OPT)
  WASM_AS = wrap_with_valgrind(WASM_AS)
  WASM_DIS = wrap_with_valgrind(WASM_DIS)
  ASM2WASM = wrap_with_valgrind(ASM2WASM)
  WASM_SHELL = wrap_with_valgrind(WASM_SHELL)
  S2WASM = wrap_with_valgrind(S2WASM)

os.environ['BINARYEN'] = os.getcwd()

def fetch_waterfall():
  rev = open(os.path.join(options.binaryen_test, 'revision')).read().strip()
  buildername = { 'linux2':'linux',
                  'darwin':'mac',
                  'win32':'windows',
                  'cygwin':'windows' }[sys.platform]
  try:
    local_rev = open(os.path.join(options.binaryen_test, 'local-revision')).read().strip()
  except:
    local_rev = None
  if local_rev == rev: return
  # fetch it
  basename = 'wasm-binaries-' + rev + '.tbz2'
  url = '/'.join(['https://storage.googleapis.com/wasm-llvm/builds', buildername, rev, basename])
  print '(downloading waterfall %s: %s)' % (rev, url)
  downloaded = urllib2.urlopen(url).read().strip()
  fullname = os.path.join(options.binaryen_test, basename)
  open(fullname, 'wb').write(downloaded)
  print '(unpacking)'
  if os.path.exists(WATERFALL_BUILD_DIR):
    shutil.rmtree(WATERFALL_BUILD_DIR)
  os.mkdir(WATERFALL_BUILD_DIR)
  subprocess.check_call(['tar', '-xf', os.path.abspath(fullname)], cwd=WATERFALL_BUILD_DIR)
  print '(noting local revision)'
  with open(os.path.join(options.binaryen_test, 'local-revision'), 'w') as o: o.write(rev)

has_vanilla_llvm = False

def setup_waterfall():
  # if we can use the waterfall llvm, do so
  global has_vanilla_llvm
  CLANG = os.path.join(BIN_DIR, 'clang')
  print 'trying waterfall clang at', CLANG
  try:
    subprocess.check_call([CLANG, '-v'])
    has_vanilla_llvm = True
    print '...success'
  except Exception, e:
    warn('could not run vanilla LLVM from waterfall: ' + str(e) + ', looked for clang at ' + CLANG)

if options.test_waterfall:
  fetch_waterfall()
  setup_waterfall()

if options.only_prepare:
  print 'waterfall is fetched and setup, exiting since --only-prepare'
  sys.exit(0)

# external tools

try:
  subprocess.check_call([NODEJS, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except:
  NODEJS = None
  warn('no node found (did not check proper js form)')

try:
  subprocess.check_call([MOZJS, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except:
  MOZJS = None
  warn('no mozjs found (did not check native wasm support nor asm.js validation)')

try:
  subprocess.check_call([EMCC, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except:
  EMCC = None
  warn('no emcc found (did not check non-vanilla emscripten/binaryen integration)')

has_vanilla_emcc = False
try:
  subprocess.check_call([os.path.join(options.binaryen_test, 'emscripten', 'emcc'), '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  has_vanilla_emcc = True
except:
  pass

# utilities

def delete_from_orbit(filename): # removes a file if it exists, using any and all ways of doing so
  try:
    os.unlink(filename)
  except:
    pass
  if not os.path.exists(filename): return
  try:
    shutil.rmtree(filename, ignore_errors=True)
  except:
    pass
  if not os.path.exists(filename): return
  try:
    os.chmod(filename, os.stat(filename).st_mode | stat.S_IWRITE)
    def remove_readonly_and_try_again(func, path, exc_info):
      if not (os.stat(path).st_mode & stat.S_IWRITE):
        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
        func(path)
      else:
        raise
    shutil.rmtree(filename, onerror=remove_readonly_and_try_again)
  except:
    pass

def fail_with_error(msg):
  global num_failures
  try:
    num_failures += 1
    raise Exception(msg)
  except Exception, e:
    print >> sys.stderr, str(e)
    if options.abort_on_first_failure:
      raise

def fail(actual, expected):
  fail_with_error("incorrect output, diff:\n\n%s" % (''.join([a.rstrip()+'\n' for a in difflib.unified_diff(expected.split('\n'), actual.split('\n'), fromfile='expected', tofile='actual')])[:]))

def fail_if_not_identical(actual, expected):
  if expected != actual:
    fail(actual, expected)

def fail_if_not_contained(actual, expected):
  if expected not in actual:
    fail(actual, expected)

if len(requested) == 0:
  tests = sorted(os.listdir(os.path.join(options.binaryen_test)))
else:
  tests = requested[:]

if not options.interpreter:
  warn('no interpreter provided (did not test spec interpreter validation)')

if not has_vanilla_emcc:
  warn('no functional emcc submodule found')

# check utilities

def binary_format_check(wast, verify_final_result=True, wasm_as_args=['-g'], binary_suffix='.fromBinary'):
  # checks we can convert the wast to binary and back

  print '     (binary format check)'
  cmd = WASM_AS + [wast, '-o', 'a.wasm'] + wasm_as_args
  print '      ', ' '.join(cmd)
  if os.path.exists('a.wasm'): os.unlink('a.wasm')
  subprocess.check_call(cmd, stdout=subprocess.PIPE)
  assert os.path.exists('a.wasm')

  cmd = WASM_DIS + ['a.wasm', '-o', 'ab.wast']
  print '      ', ' '.join(cmd)
  if os.path.exists('ab.wast'): os.unlink('ab.wast')
  subprocess.check_call(cmd, stdout=subprocess.PIPE)
  assert os.path.exists('ab.wast')

  # make sure it is a valid wast
  cmd = WASM_OPT + ['ab.wast']
  print '      ', ' '.join(cmd)
  subprocess.check_call(cmd, stdout=subprocess.PIPE)

  if verify_final_result:
    expected = open(wast + binary_suffix).read()
    actual = open('ab.wast').read()
    if actual != expected:
      fail(actual, expected)

  return 'ab.wast'

def minify_check(wast, verify_final_result=True):
  # checks we can parse minified output

  print '     (minify check)'
  cmd = WASM_OPT + [wast, '--print-minified']
  print '      ', ' '.join(cmd)
  subprocess.check_call(WASM_OPT + [wast, '--print-minified'], stdout=open('a.wasm', 'w'), stderr=subprocess.PIPE)
  assert os.path.exists('a.wasm')
  subprocess.check_call(WASM_OPT + ['a.wasm', '--print-minified'], stdout=open('b.wasm', 'w'), stderr=subprocess.PIPE)
  assert os.path.exists('b.wasm')
  if verify_final_result:
    expected = open('a.wasm').read()
    actual = open('b.wasm').read()
    if actual != expected:
      fail(actual, expected)
  if os.path.exists('a.wasm'): os.unlink('a.wasm')
  if os.path.exists('b.wasm'): os.unlink('b.wasm')

# tests

print '[ checking --help is useful... ]\n'

not_executable_suffix = ['.txt', '.js', '.ilk', '.pdb', '.dll']
executables = sorted(filter(lambda x: not any(x.endswith(s) for s in
                                              not_executable_suffix) and os.path.isfile(x),
                            os.listdir(options.binaryen_bin)))
for e in executables:
  print '.. %s --help' % e
  out, err = subprocess.Popen([os.path.join(options.binaryen_bin, e), '--help'],
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE).communicate()
  assert len(out) == 0, 'Expected no stdout, got:\n%s' % out
  assert e.replace('.exe', '') in err, 'Expected help to contain program name, got:\n%s' % err
  assert len(err.split('\n')) > 8, 'Expected some help, got:\n%s' % err

print '\n[ checking wasm-opt -o notation... ]\n'

wast = os.path.join(options.binaryen_test, 'hello_world.wast')
delete_from_orbit('a.wast')
cmd = WASM_OPT + [wast, '-o', 'a.wast']
run_command(cmd)
fail_if_not_identical(open('a.wast').read(), open(wast).read())

print '\n[ checking wasm-opt passes... ]\n'

for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'passes'))):
  if t.endswith('.wast'):
    print '..', t
    passname = os.path.basename(t).replace('.wast', '')
    opts = ['-' + passname] if passname.startswith('O') else ['--' + p for p in passname.split('_')]
    t = os.path.join(options.binaryen_test, 'passes', t)
    actual = ''
    for module, asserts in split_wast(t):
      assert len(asserts) == 0
      with open('split.wast', 'w') as o: o.write(module)
      cmd = WASM_OPT + opts + ['split.wast', '--print']
      actual += run_command(cmd)
      # also check debug mode output is valid
      debugged = run_command(cmd + ['--debug'], stderr=subprocess.PIPE)
      fail_if_not_contained(actual, debugged)
    fail_if_not_identical(actual, open(os.path.join(options.binaryen_test, 'passes', passname + '.txt'), 'rb').read())

print '[ checking asm2wasm testcases... ]\n'

for asm in tests:
  if asm.endswith('.asm.js'):
    for precise in [1, 0]:
      for opts in [1, 0]:
        cmd = ASM2WASM + [os.path.join(options.binaryen_test, asm)]
        wasm = asm.replace('.asm.js', '.fromasm')
        if not precise:
          cmd += ['--imprecise']
          wasm += '.imprecise'
        if not opts:
          wasm += '.no-opts'
          if precise:
            cmd += ['-O0'] # test that -O0 does nothing
        else:
          cmd += ['-O']
        if precise and opts:
          # test mem init importing
          open('a.mem', 'wb').write(asm)
          cmd += ['--mem-init=a.mem']
          if asm[0] == 'e':
            cmd += ['--mem-base=1024']
        if 'i64' in asm or 'wasm-only' in asm:
          cmd += ['--wasm-only']
        wasm = os.path.join(options.binaryen_test, wasm)
        print '..', asm, wasm
        actual = run_command(cmd)

        # verify output
        if not os.path.exists(wasm):
          fail_with_error('output .wast file %s does not exist' % wasm)
        expected = open(wasm, 'rb').read()
        if actual != expected:
          fail(actual, expected)

        binary_format_check(wasm, verify_final_result=False)

        # verify in wasm
        if options.interpreter:
          # remove imports, spec interpreter doesn't know what to do with them
          subprocess.check_call(WASM_OPT + ['--remove-imports', wasm], stdout=open('ztemp.wast', 'w'), stderr=subprocess.PIPE)
          proc = subprocess.Popen([options.interpreter, 'ztemp.wast'], stderr=subprocess.PIPE)
          out, err = proc.communicate()
          if proc.returncode != 0:
            try: # to parse the error
              reported = err.split(':')[1]
              start, end = reported.split('-')
              start_line, start_col = map(int, start.split('.'))
              lines = open('ztemp.wast').read().split('\n')
              print
              print '='*80
              print lines[start_line-1]
              print (' '*(start_col-1)) + '^'
              print (' '*(start_col-2)) + '/_\\'
              print '='*80
              print err
            except Exception, e:
              fail_with_error('wasm interpreter error: ' + err) # failed to pretty-print
            fail_with_error('wasm interpreter error')

print '\n[ checking wasm-opt parsing & printing... ]\n'

for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'print'))):
  if t.endswith('.wast'):
    print '..', t
    wasm = os.path.basename(t).replace('.wast', '')
    cmd = WASM_OPT + [os.path.join(options.binaryen_test, 'print', t), '--print']
    print '    ', ' '.join(cmd)
    actual, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
    fail_if_not_identical(actual, open(os.path.join(options.binaryen_test, 'print', wasm + '.txt')).read())
    cmd = WASM_OPT + [os.path.join(options.binaryen_test, 'print', t), '--print-minified']
    print '    ', ' '.join(cmd)
    actual, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
    fail_if_not_identical(actual.strip(), open(os.path.join(options.binaryen_test, 'print', wasm + '.minified.txt')).read().strip())

print '\n[ checking wasm-opt testcases... ]\n'

for t in tests:
  if t.endswith('.wast') and not t.startswith('spec'):
    print '..', t
    t = os.path.join(options.binaryen_test, t)
    cmd = WASM_OPT + [t, '--print']
    actual = run_command(cmd)
    actual = actual.replace('printing before:\n', '')

    expected = open(t, 'rb').read()
    if actual != expected:
      fail(actual, expected)

    binary_format_check(t, wasm_as_args=['-g']) # test with debuginfo
    binary_format_check(t, wasm_as_args=[], binary_suffix='.fromBinary.noDebugInfo') # test without debuginfo

    minify_check(t)

print '\n[ checking wasm-shell spec testcases... ]\n'

if len(requested) == 0:
  BLACKLIST = ['memory.wast', 'binary.wast'] # FIXME we support old and new memory formats, for now, until 0xc, and so can't pass this old-style test.
  # FIXME to update the spec to 0xd, we need to implement (register "name") for import.wast
  spec_tests = [os.path.join('spec', t) for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'spec'))) if t not in BLACKLIST]
else:
  spec_tests = requested[:]

for t in spec_tests:
  if t.startswith('spec') and t.endswith('.wast'):
    print '..', t
    wast = os.path.join(options.binaryen_test, t)

    # skip checks for some tests
    if os.path.basename(wast) in ['linking.wast', 'nop.wast', 'stack.wast', 'typecheck.wast', 'unwind.wast']: # FIXME
      continue

    def run_spec_test(wast):
      cmd = WASM_SHELL + [wast]
      # we must skip the stack machine portions of spec tests or apply other extra args
      extra = {
      }
      cmd = cmd + (extra.get(os.path.basename(wast)) or [])
      return run_command(cmd, stderr=subprocess.PIPE)

    def check_expected(actual, expected):
      if expected and os.path.exists(expected):
        expected = open(expected).read()
        # fix it up, our pretty (i32.const 83) must become compared to a homely 83 : i32
        def fix(x):
          x = x.strip()
          if not x: return x
          v, t = x.split(' : ')
          if v.endswith('.'): v = v[:-1] # remove trailing '.'
          return '(' + t + '.const ' + v + ')'
        expected = '\n'.join(map(fix, expected.split('\n')))
        print '       (using expected output)'
        actual = actual.strip()
        expected = expected.strip()
        if actual != expected:
          fail(actual, expected)

    expected = os.path.join(options.binaryen_test, 'spec', 'expected-output', os.path.basename(wast) + '.log')

    # some spec tests should fail (actual process failure, not just assert_invalid)
    try:
      actual = run_spec_test(wast)
    except Exception, e:
      if ('wasm-validator error' in str(e) or 'parse exception' in str(e)) and '.fail.' in t:
        print '<< test failed as expected >>'
        continue # don't try all the binary format stuff TODO
      else:
        fail_with_error(str(e))

    check_expected(actual, expected)

    # skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature
    if os.path.basename(wast) in ['exports.wast']: # FIXME
      continue

    # we must ignore some binary format splits
    splits_to_skip = {
        'func.wast': [2],
        'return.wast': [2]
    }

    # check binary format. here we can verify execution of the final result, no need for an output verification
    split_num = 0
    if os.path.basename(wast) not in []: # avoid some tests with things still being sorted out in the spec
      actual = ''
      for module, asserts in split_wast(wast):
        skip = splits_to_skip.get(os.path.basename(wast)) or []
        if split_num in skip:
          print '    skipping split module', split_num - 1
          split_num += 1
          continue
        print '    testing split module', split_num
        split_num += 1
        with open('split.wast', 'w') as o: o.write(module + '\n' + '\n'.join(asserts))
        run_spec_test('split.wast') # before binary stuff - just check it's still ok split out
        result_wast = binary_format_check('split.wast', verify_final_result=False)
        # add the asserts, and verify that the test still passes
        open(result_wast, 'a').write('\n' + '\n'.join(asserts))
        actual += run_spec_test(result_wast)
      # compare all the outputs to the expected output
      check_expected(actual, os.path.join(options.binaryen_test, 'spec', 'expected-output', os.path.basename(wast) + '.log'))

if NODEJS:
  print '\n[ checking binaryen.js testcases... ]\n'

  for s in sorted(os.listdir(os.path.join(options.binaryen_test, 'binaryen.js'))):
    if not s.endswith('.js'): continue
    print s
    f = open('a.js', 'w')
    f.write(open(os.path.join(options.binaryen_bin, 'binaryen.js')).read())
    f.write(open(os.path.join(options.binaryen_test, 'binaryen.js', s)).read())
    f.close()
    cmd = [NODEJS, 'a.js']
    out = run_command(cmd)
    expected = open(os.path.join(options.binaryen_test, 'binaryen.js', s + '.txt')).read()
    if expected not in out:
      fail(out, expected)

print '\n[ checking .s testcases... ]\n'

for dot_s_dir in ['dot_s', 'llvm_autogenerated']:
  for s in sorted(os.listdir(os.path.join(options.binaryen_test, dot_s_dir))):
    if not s.endswith('.s'): continue
    print '..', s
    wasm = s.replace('.s', '.wast')
    full = os.path.join(options.binaryen_test, dot_s_dir, s)
    stack_alloc = ['--allocate-stack=1024'] if dot_s_dir == 'llvm_autogenerated' else []
    cmd = S2WASM + [full, '--emscripten-glue'] + stack_alloc
    if s.startswith('start_'):
      cmd.append('--start')
    actual = run_command(cmd)

    # verify output
    expected_file = os.path.join(options.binaryen_test, dot_s_dir, wasm)
    if not os.path.exists(expected_file):
      print actual
      fail_with_error('output ' + expected_file + ' does not exist')
    expected = open(expected_file, 'rb').read()
    if actual != expected:
      fail(actual, expected)

    # verify with options
    cmd = S2WASM + [full, '--global-base=1024'] + stack_alloc
    run_command(cmd)

    # run wasm-shell on the .wast to verify that it parses
    cmd = WASM_SHELL + [expected_file]
    run_command(cmd)

print '\n[ running linker tests... ]\n'
# The {main,foo,bar,baz}.s files were created by running clang over the respective
# c files. The foobar.bar archive was created by running:
# llvm-ar -format=gnu rc foobar.a quux.s foo.s bar.s baz.s
cmd = S2WASM + [os.path.join(options.binaryen_test, 'linker', 'main.s'), '-l', os.path.join(options.binaryen_test, 'linker', 'archive', 'foobar.a')]
output = run_command(cmd)
# foo should come from main.s and return 42
fail_if_not_contained(output, '(func $foo')
fail_if_not_contained(output, '(i32.const 42)')
# bar should be linked in from bar.s
fail_if_not_contained(output, '(func $bar')
# quux should be linked in from bar.s even though it comes before bar.s in the archive
fail_if_not_contained(output, '(func $quux')
# baz should not be linked in at all
if 'baz' in output:
  fail_with_error('output should not contain "baz": ' + output)

# Test an archive using a string table
cmd = S2WASM + [os.path.join(options.binaryen_test, 'linker', 'main.s'), '-l', os.path.join(options.binaryen_test, 'linker', 'archive', 'barlong.a')]
output = run_command(cmd)
# bar should be linked from the archive
fail_if_not_contained(output, '(func $bar')

# Test exporting memory growth function
cmd = S2WASM + [os.path.join(options.binaryen_test, 'linker', 'main.s'), '--emscripten-glue', '--allow-memory-growth']
output = run_command(cmd)
fail_if_not_contained(output, '(export "__growWasmMemory" (func $__growWasmMemory))')
fail_if_not_contained(output, '(func $__growWasmMemory (param $newSize i32)')

print '\n[ running validation tests... ]\n'
# Ensure the tests validate by default
cmd = WASM_AS + [os.path.join(options.binaryen_test, 'validator', 'invalid_export.wast')]
run_command(cmd)
cmd = WASM_AS + [os.path.join(options.binaryen_test, 'validator', 'invalid_import.wast')]
run_command(cmd)
cmd = WASM_AS + ['--validate=web', os.path.join(options.binaryen_test, 'validator', 'invalid_export.wast')]
run_command(cmd, expected_status=1)
cmd = WASM_AS + ['--validate=web', os.path.join(options.binaryen_test, 'validator', 'invalid_import.wast')]
run_command(cmd, expected_status=1)
cmd = WASM_AS + ['--validate=none', os.path.join(options.binaryen_test, 'validator', 'invalid_return.wast')]
run_command(cmd)

if options.torture and options.test_waterfall:

  print '\n[ checking torture testcases... ]\n'

  unexpected_result_count = 0

  import test.waterfall.src.link_assembly_files as link_assembly_files
  s2wasm_torture_out = os.path.abspath(os.path.join(options.binaryen_test, 's2wasm-torture-out'))
  if os.path.isdir(s2wasm_torture_out):
    shutil.rmtree(s2wasm_torture_out)
  os.mkdir(s2wasm_torture_out)
  unexpected_result_count += link_assembly_files.run(
      linker=os.path.abspath(S2WASM_EXE),
      files=os.path.abspath(os.path.join(options.binaryen_test, 'torture-s', '*.s')),
      fails=os.path.abspath(os.path.join(options.binaryen_test, 's2wasm_known_gcc_test_failures.txt')),
      out=s2wasm_torture_out)
  assert os.path.isdir(s2wasm_torture_out), 'Expected output directory %s' % s2wasm_torture_out

  import test.waterfall.src.execute_files as execute_files
  unexpected_result_count += execute_files.run(
      runner=os.path.abspath(WASM_SHELL_EXE),
      files=os.path.abspath(os.path.join(s2wasm_torture_out, '*.wast')),
      fails=os.path.abspath(os.path.join(options.binaryen_test, 's2wasm_known_binaryen_shell_test_failures.txt')),
      out='',
      wasmjs='')

  shutil.rmtree(s2wasm_torture_out)
  if unexpected_result_count:
    fail('%s failures' % unexpected_result_count, '0 failures')

if has_vanilla_emcc and has_vanilla_llvm and 0:

  print '\n[ checking emcc WASM_BACKEND testcases...]\n'

  try:
    if has_vanilla_llvm:
      os.environ['LLVM'] = BIN_DIR # use the vanilla LLVM
    else:
      # if we did not set vanilla llvm, then we must set this env var to make emcc use the wasm backend.
      # (if we are using vanilla llvm, things should just work)
      print '(not using vanilla llvm, so setting env var to tell emcc to use wasm backend)'
      os.environ['EMCC_WASM_BACKEND'] = '1'
    VANILLA_EMCC = os.path.join(options.binaryen_test, 'emscripten', 'emcc')
    # run emcc to make sure it sets itself up properly, if it was never run before
    command = [VANILLA_EMCC, '-v']
    print '____' + ' '.join(command)
    subprocess.check_call(command)

    for c in sorted(os.listdir(os.path.join(options.binaryen_test, 'wasm_backend'))):
      if not c.endswith('cpp'): continue
      print '..', c
      base = c.replace('.cpp', '').replace('.c', '')
      expected = open(os.path.join(options.binaryen_test, 'wasm_backend', base + '.txt')).read()
      for opts in [[], ['-O1'], ['-O2']]:
        only = [] if opts != ['-O1'] or '_only' not in base else ['-s', 'ONLY_MY_CODE=1'] # only my code is a hack we used early in wasm backend dev, which somehow worked, but only with -O1
        command = [VANILLA_EMCC, '-o', 'a.wasm.js', os.path.join(options.binaryen_test, 'wasm_backend', c)] + opts + only
        print '....' + ' '.join(command)
        if os.path.exists('a.wasm.js'): os.unlink('a.wasm.js')
        subprocess.check_call(command)
        if NODEJS:
          print '  (check in node)'
          cmd = [NODEJS, 'a.wasm.js']
          out = run_command(cmd)
          if out.strip() != expected.strip():
            fail(out, expected)
  finally:
    if has_vanilla_llvm:
      del os.environ['LLVM']
    else:
      del os.environ['EMCC_WASM_BACKEND']

print '\n[ checking example testcases... ]\n'

if options.run_gcc_tests:
  print '\n[ checking native gcc testcases...]\n'  
  if not NATIVECC or not NATIVEXX:
    fail_with_error('Native compiler (e.g. gcc/g++) was not found in PATH!')
  else:
    for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'example'))):
      output_file = os.path.join(options.binaryen_bin, 'example')
      cmd = ['-I' + os.path.join(options.binaryen_root, 'src'), '-g', '-lasmjs', '-lsupport', '-L' + os.path.join(options.binaryen_bin, '..', 'lib'), '-pthread', '-o', output_file]
      if t.endswith('.txt'):
        # check if there is a trace in the file, if so, we should build it
        out = subprocess.Popen([os.path.join('scripts', 'clean_c_api_trace.py'), os.path.join(options.binaryen_test, 'example', t)], stdout=subprocess.PIPE).communicate()[0]
        if len(out) == 0:
          print '  (no trace in ', t, ')'
          continue
        print '  (will check trace in ', t, ')'
        src = 'trace.cpp'
        with open(src, 'w') as o: o.write(out)
        expected = os.path.join(options.binaryen_test, 'example', t + '.txt')
      else:
        src = os.path.join(options.binaryen_test, 'example', t)
        expected = os.path.join(options.binaryen_test, 'example', '.'.join(t.split('.')[:-1]) + '.txt')
      if src.endswith(('.c', '.cpp')):
        # build the C file separately
        extra = [NATIVECC, src, '-c', '-o', 'example.o',
                 '-I' + os.path.join(options.binaryen_root, 'src'), '-g', '-L' + os.path.join(options.binaryen_bin, '..', 'lib'), '-pthread']
        print 'build: ', ' '.join(extra)
        subprocess.check_call(extra)
        # Link against the binaryen C library DSO, using an executable-relative rpath
        cmd = ['example.o', '-lbinaryen'] + cmd + ['-Wl,-rpath=$ORIGIN/../lib']
      else:
        continue
      print '  ', t, src, expected
      if os.environ.get('COMPILER_FLAGS'):
        for f in os.environ.get('COMPILER_FLAGS').split(' '):
          cmd.append(f)
      cmd = [NATIVEXX, '-std=c++11'] + cmd
      try:
        print 'link: ', ' '.join(cmd)
        subprocess.check_call(cmd)
        print 'run...', output_file
        proc = subprocess.Popen([output_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        actual, err = proc.communicate()
        assert proc.returncode == 0, [proc.returncode, actual, err]
      finally:
        os.remove(output_file)
        if sys.platform == 'darwin':
          # Also removes debug directory produced on Mac OS
          shutil.rmtree(output_file + '.dSYM')

      expected = open(expected).read()
      if actual != expected:
        fail(actual, expected)

if EMCC:

  if MOZJS and 0:

    print '\n[ checking native wasm support ]\n'

    command = [EMCC, '-o', 'a.wasm.js', '-s', 'BINARYEN=1', os.path.join(options.binaryen_test, 'hello_world.c'), '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_SCRIPTS="spidermonkify.py"']
    print ' '.join(command)
    subprocess.check_call(command)

    cmd = [MOZJS, 'a.wasm.js']
    out = run_command(cmd)
    assert 'hello, world!' in out, out

    proc = subprocess.Popen([NODEJS, 'a.wasm.js'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate()
    assert proc.returncode != 0, 'should fail on no wasm support'
    assert 'no native wasm support detected' in err, err

  print '\n[ checking wasm.js methods... ]\n'

  for method_init in ['interpret-asm2wasm', 'interpret-s-expr', 'asmjs', 'interpret-binary', 'asmjs,interpret-binary', 'interpret-binary,asmjs']:
    # check success and failure for simple modes, only success for combined/fallback ones
    for success in [1, 0] if ',' not in method_init else [1]:
      method = method_init
      command = [EMCC, '-o', 'a.wasm.js', '-s', 'BINARYEN=1', os.path.join(options.binaryen_test, 'hello_world.c') ]
      command += ['-s', 'BINARYEN_METHOD="' + method + '"']
      print method, ' : ', ' '.join(command), ' => ', success
      subprocess.check_call(command)

      see_polyfill =  'var WasmJS = ' in open('a.wasm.js').read()

      if method and 'interpret' not in method:
        assert not see_polyfill, 'verify polyfill was not added - we specified a method, and it does not need it'
      else:
        assert see_polyfill, 'we need the polyfill'

      def break_cashew():
        asm = open('a.wasm.asm.js').read()
        asm = asm.replace('"almost asm"', '"use asm"; var not_in_asm = [].length + (true || { x: 5 }.x);')
        asm = asm.replace("'almost asm'", '"use asm"; var not_in_asm = [].length + (true || { x: 5 }.x);')
        with open('a.wasm.asm.js', 'w') as o: o.write(asm)
      if method.startswith('interpret-asm2wasm'):
        os.unlink('a.wasm.wast') # we should not need the .wast
        if not success:
          break_cashew() # we need cashew
      elif method.startswith('interpret-s-expr'):
        os.unlink('a.wasm.asm.js') # we should not need the .asm.js
        if not success:
          os.unlink('a.wasm.wast')
      elif method.startswith('asmjs'):
        os.unlink('a.wasm.wast') # we should not need the .wast
        break_cashew() # we don't use cashew, so ok to break it
        if not success:
          os.unlink('a.wasm.js')
      elif method.startswith('interpret-binary'):
        os.unlink('a.wasm.wast') # we should not need the .wast
        os.unlink('a.wasm.asm.js') # we should not need the .asm.js
        if not success:
          os.unlink('a.wasm.wasm')
      else:
        1/0
      if NODEJS:
        proc = subprocess.Popen([NODEJS, 'a.wasm.js'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = proc.communicate()
        if success:
          assert proc.returncode == 0, err
          assert 'hello, world!' in out, out
        else:
          assert proc.returncode != 0, err
          assert 'hello, world!' not in out, out

  print '\n[ checking wasm.js testcases... ]\n'

  for c in tests:
    if c.endswith(('.c', '.cpp')):
      print '..', c
      base = c.replace('.cpp', '').replace('.c', '')
      post = base + '.post.js'
      try:
        post = open(os.path.join(options.binaryen_test, post)).read()
      except:
        post = None
      expected = open(os.path.join(options.binaryen_test, base + '.txt')).read()
      emcc = os.path.join(options.binaryen_test, base + '.emcc')
      extra = []
      if os.path.exists(emcc):
        extra = json.loads(open(emcc).read())
      if os.path.exists('a.normal.js'): os.unlink('a.normal.js')
      for opts in [[], ['-O1'], ['-O2'], ['-O3'], ['-Oz']]:
        for method in ['interpret-asm2wasm', 'interpret-s-expr', 'interpret-binary']:
          command = [EMCC, '-o', 'a.wasm.js', '-s', 'BINARYEN=1', os.path.join(options.binaryen_test, c)] + opts + extra
          command += ['-s', 'BINARYEN_METHOD="' + method + '"']
          print '....' + ' '.join(command)
          subprocess.check_call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
          if post:
            open('a.wasm.js', 'a').write(post)
          else:
            print '     (no post)'
          for which in ['wasm']:
            print '......', which
            try:
              args = json.loads(open(os.path.join(options.binaryen_test, base + '.args')).read())
            except:
              args = []
              print '     (no args)'

            def execute():
              if NODEJS:
                cmd = [NODEJS, 'a.' + which + '.js'] + args
                out = run_command(cmd)
                if out.strip() != expected.strip():
                  fail(out, expected)

            execute()

            if method in ['interpret-asm2wasm', 'interpret-s-expr']:
              # binary and back
              shutil.copyfile('a.wasm.wast', 'a.wasm.original.wast')
              recreated = binary_format_check('a.wasm.wast', verify_final_result=False)
              shutil.copyfile(recreated, 'a.wasm.wast')
              execute()

if num_failures == 0:
  print '\n[ success! ]'

if warnings:
  print '\n' + '\n'.join(warnings)

if num_failures > 0:
  print '\n[ ' + str(num_failures) + ' failures! ]'

sys.exit(num_failures)