diff options
| -rw-r--r-- | Makefile.am | 28 | ||||
| -rwxr-xr-x | test/ConfirmTests.py | 58 | ||||
| -rwxr-xr-x | test/GenerateTests.py | 117 | ||||
| -rwxr-xr-x | test/LedgerHarness.py | 116 | ||||
| -rwxr-xr-x | test/RegressTests.py | 78 | ||||
| -rwxr-xr-x | test/fullcheck.sh | 3 | ||||
| -rwxr-xr-x | tools/times.sh | 6 |
7 files changed, 221 insertions, 185 deletions
diff --git a/Makefile.am b/Makefile.am index 168687f8..b295fa35 100644 --- a/Makefile.am +++ b/Makefile.am @@ -359,7 +359,7 @@ RegressTests_SOURCES = test/RegressTests.py EXTRA_DIST += test/regress test/convert.py RegressTests: $(srcdir)/test/RegressTests.py - echo "$(PYTHON) $(srcdir)/test/RegressTests.py $(top_builddir)/ledger$(EXEEXT) $(srcdir)/test/regress" > $@ + echo "$(PYTHON) $(srcdir)/test/RegressTests.py $(top_builddir)/ledger$(EXEEXT) $(srcdir)/test/regress \"\$$@\"" > $@ chmod 755 $@ BaselineTests_SOURCES = test/RegressTests.py @@ -367,7 +367,7 @@ BaselineTests_SOURCES = test/RegressTests.py EXTRA_DIST += test/baseline BaselineTests: $(srcdir)/test/RegressTests.py - echo "$(PYTHON) $(srcdir)/test/RegressTests.py $(top_builddir)/ledger$(EXEEXT) $(srcdir)/test/baseline" > $@ + echo "$(PYTHON) $(srcdir)/test/RegressTests.py $(top_builddir)/ledger$(EXEEXT) $(srcdir)/test/baseline \"\$$@\"" > $@ chmod 755 $@ ConfirmTests_SOURCES = test/ConfirmTests.py @@ -375,29 +375,35 @@ ConfirmTests_SOURCES = test/ConfirmTests.py EXTRA_DIST += test/input ConfirmTests: $(srcdir)/test/ConfirmTests.py - echo "$(PYTHON) $(srcdir)/test/ConfirmTests.py $(top_builddir)/ledger$(EXEEXT) $(srcdir)/test/input" > $@ + echo "$(PYTHON) $(srcdir)/test/ConfirmTests.py $(top_builddir)/ledger$(EXEEXT) $(srcdir)/test/input \"\$$@\"" > $@ chmod 755 $@ GenerateTests_SOURCES = test/GenerateTests.py GenerateTests: $(srcdir)/test/GenerateTests.py - echo "$(PYTHON) $(srcdir)/test/GenerateTests.py $(top_builddir)/ledger$(EXEEXT) $(srcdir)/test/input" > $@ + echo "$(PYTHON) $(srcdir)/test/GenerateTests.py $(top_builddir)/ledger$(EXEEXT) 1 20 \"\$$@\"" > $@ chmod 755 $@ FULLCHECK=$(srcdir)/test/fullcheck.sh if HAVE_CPPUNIT -fullcheck: $(TESTS) - sh $(FULLCHECK) $(top_builddir)/UtilTests$(EXEEXT) --verify - sh $(FULLCHECK) $(top_builddir)/MathTests$(EXEEXT) --verify - sh $(FULLCHECK) $(top_builddir)/ExprTests$(EXEEXT) --verify - sh $(FULLCHECK) $(top_builddir)/DataTests$(EXEEXT) --verify - sh $(FULLCHECK) $(top_builddir)/ReportTests$(EXEEXT) --verify +cppunittests: $(TESTS) + @sh $(FULLCHECK) $(top_builddir)/UtilTests$(EXEEXT) --verify + @sh $(FULLCHECK) $(top_builddir)/MathTests$(EXEEXT) --verify + @sh $(FULLCHECK) $(top_builddir)/ExprTests$(EXEEXT) --verify + @sh $(FULLCHECK) $(top_builddir)/DataTests$(EXEEXT) --verify + @sh $(FULLCHECK) $(top_builddir)/ReportTests$(EXEEXT) --verify else -fullcheck: check +cppunittests: $(TESTS) @test 1 -eq 1 endif +fullcheck: cppunittests + @$(top_builddir)/RegressTests --verify --gmalloc + @$(top_builddir)/BaselineTests --verify --gmalloc + @$(top_builddir)/ConfirmTests --verify --gmalloc + @$(top_builddir)/GenerateTests --verify --gmalloc + ###################################################################### EXTRA_DIST += doc/LICENSE doc/NEWS doc/README diff --git a/test/ConfirmTests.py b/test/ConfirmTests.py index 4b64af7c..0b3d4897 100755 --- a/test/ConfirmTests.py +++ b/test/ConfirmTests.py @@ -7,7 +7,13 @@ import sys import os import re -from subprocess import Popen, PIPE +from LedgerHarness import LedgerHarness + +harness = LedgerHarness(sys.argv) +tests = sys.argv[2] + +if not os.path.isdir(tests) and not os.path.isfile(tests): + sys.exit(1) commands = [ "-f '$tests/standard.dat' -O 0ecbb1b15e2cf3e515cc0f8533e5bb0fb2326728", @@ -19,16 +25,6 @@ commands = [ "-f '$tests/standard.dat' -G c0226fafdf9e6711ac9121cf263e2d50791859cb" ] -ledger = sys.argv[1] -tests = sys.argv[2] -succeeded = 0 -failed = 0 - -if not os.path.isfile(ledger): - sys.exit(1) -if not os.path.isdir(tests) and not os.path.isfile(tests): - sys.exit(1) - def clean(num): num = re.sub("(\s+|\$|,)","", num) m = re.search("([-0-9.]+)", num) @@ -37,26 +33,25 @@ def clean(num): else: return float(num) -def confirm_report(args): +def confirm_report(command): index = 1 last_line = "" failure = False running_total = 0.0 - p = Popen(re.sub('\$cmd', 'reg', args), shell=True, - stdin=PIPE, stdout=PIPE, stderr=PIPE, - close_fds=True) + p = harness.run(re.sub('\$cmd', 'reg', command)) - for line in p.stdout.readlines(): + for line in harness.readlines(p.stdout): match = re.match("\\s*([-$,0-9.]+)\\s+([-$,0-9.]+)", line[54:]) if not match: continue + value = clean(match.group(1)) total = clean(match.group(2)) - running_total += value + diff = abs(running_total - total) - if re.search(' -[VGB] ', args) and diff < 0.015: + if re.search(' -[VGB] ', command) and diff < 0.015: diff = 0.0 if diff > 0.001: print "DISCREPANCY: %.3f (%.3f - %.3f) at line %d:" % \ @@ -70,16 +65,14 @@ def confirm_report(args): balance_total = 0.0 - p = Popen(re.sub('\$cmd', 'bal', args), shell=True, - stdin=PIPE, stdout=PIPE, stderr=PIPE, - close_fds=True) + p = harness.run(re.sub('\$cmd', 'bal', command)) - for line in p.stdout.readlines(): + for line in harness.readlines(p.stdout): if line[0] != '-': balance_total = clean(line[:20]) diff = abs(balance_total - running_total) - if re.search(' -[VGB] ', args) and diff < 0.015: + if re.search(' -[VGB] ', command) and diff < 0.015: diff = 0.0 if diff > 0.001: print @@ -91,20 +84,9 @@ def confirm_report(args): return not failure for cmd in commands: - if confirm_report("%s --args-only --verify --columns=80 %s" % - (ledger, re.sub('\$tests', tests, cmd))): - sys.stdout.write(".") - succeeded += 1 + if confirm_report('$ledger $cmd ' + re.sub('\$tests', tests, cmd)): + harness.success() else: - sys.stdout.write("E") - failed += 1 - -print -if succeeded > 0: - print "OK (%d) " % succeeded, -if failed > 0: - print "FAILED (%d)" % failed, -print - -sys.exit(failed) + harness.failure() +harness.exit() diff --git a/test/GenerateTests.py b/test/GenerateTests.py index 88c223cd..aa36737d 100755 --- a/test/GenerateTests.py +++ b/test/GenerateTests.py @@ -4,68 +4,44 @@ # final balance is the same as what the balance report shows. import sys -import os -import re +#import re -from subprocess import Popen, PIPE from difflib import ndiff -ledger = sys.argv[1] -succeeded = 0 -failed = 0 - -if not os.path.isfile(ledger): - sys.exit(1) - -def normalize(line): - match = re.match("((\s*)([A-Za-z]+)?(\s*)([-0-9.]+)(\s*)([A-Za-z]+)?)( (.+))?$", line) - if match: - if match.group(3): - prefix = match.group(3) + " " + match.group(5) - if match.group(8): - return prefix + match.group(8) - return prefix - elif match.group(7): - prefix = match.group(7) + " " + match.group(5) - if match.group(8): - return prefix + match.group(8) - return prefix - return line +from LedgerHarness import LedgerHarness + +harness = LedgerHarness(sys.argv) + +#def normalize(line): +# match = re.match("((\s*)([A-Za-z]+)?(\s*)([-0-9.]+)(\s*)([A-Za-z]+)?)( (.+))?$", line) +# if match: +# if match.group(3): +# prefix = match.group(3) + " " + match.group(5) +# if match.group(8): +# return prefix + match.group(8) +# return prefix +# elif match.group(7): +# prefix = match.group(7) + " " + match.group(5) +# if match.group(8): +# return prefix + match.group(8) +# return prefix +# return line def generation_test(seed): - global succeeded, failed - - p_gen = Popen("%s --args-only --actual --seed=%d generate" % (ledger, seed), - shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, - close_fds=True) - cout = "" - cout_data = os.read(p_gen.stdout.fileno(), 8192) - while cout_data: - if cout_data: - cout += cout_data - cout_data = os.read(p_gen.stdout.fileno(), 8192) - if cout_data: - cout += cout_data - - if p_gen.wait() != 0: - print "Generation for seed %d failed due to error:" % seed - print p_gen.stderr.read() - del p_gen + p_gen = harness.run('$ledger --seed=%d generate' % seed) + + cout = harness.read(p_gen.stdout) + + if not harness.wait(p_gen, msg=("Generation for seed %d failed:" % seed)): return False - del p_gen - p_print = Popen("%s --args-only --actual -f - print" % ledger, shell=True, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) + p_print = harness.run('$ledger --actual -f - print') p_print.stdin.write(cout) p_print.stdin.close() p_print_out = p_print.stdout.read() - if p_print.wait() != 0: - print "Print for seed %d failed due to error:" % seed - print p_print.stderr.read() - del p_print + if not harness.wait(p_print, msg=("Print for seed %d failed:" % seed)): return False - del p_print #p_cerr_bal = Popen("%s --args-only -f - bal" % ledger, shell=True, # stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) @@ -81,34 +57,24 @@ def generation_test(seed): # return False #del p_cerr_bal - p_cout_bal = Popen("%s --args-only -f - bal" % ledger, shell=True, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) + p_cout_bal = harness.run('$ledger -f - bal') p_cout_bal.stdin.write(cout) p_cout_bal.stdin.close() - cout_lines = p_cout_bal.stdout.readlines() - norm_cout_lines = [normalize(line) for line in cout_lines] + cout_lines = harness.readlines(p_cout_bal.stdout) + #norm_cout_lines = [normalize(line) for line in cout_lines] - if p_cout_bal.wait() != 0: - print "Stdout balance for seed %d failed due to error:" % seed - print p_cout_bal.stderr.read() - del p_cout_bal + if not harness.wait(p_cout_bal, msg=("Stdout balance for seed %d failed:" % seed)): return False - del p_cout_bal - p_print_bal = Popen("%s --args-only -f - bal" % ledger, shell=True, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) + p_print_bal = harness.run('$ledger -f - bal') p_print_bal.stdin.write(p_print_out) p_print_bal.stdin.close() - print_lines = p_print_bal.stdout.readlines() + print_lines = harness.readlines(p_print_bal.stdout) - if p_print_bal.wait() != 0: - print "Print balance for seed %d failed due to error:" % seed - print p_print_bal.stderr.read() - del p_print_bal + if not harness.wait(p_print_bal, msg=("Print balance for seed %d failed:" % seed)): return False - del p_print_bal success = True #printed = False @@ -130,7 +96,6 @@ def generation_test(seed): if not printed: if success: print print "Generation failure in output from seed %d (cout vs. print):" % seed - if success: failed += 1 success = False printed = True print " ", line @@ -145,18 +110,8 @@ if len(sys.argv) > 3: for i in range(beg_range, end_range): if generation_test(i): - sys.stdout.write(".") - succeeded += 1 + harness.success() else: - sys.stdout.write("E") - failed += 1 - -print -if succeeded > 0: - print "OK (%d) " % succeeded, -if failed > 0: - print "FAILED (%d)" % failed, -print - -sys.exit(failed) + harness.failure() +harness.exit() diff --git a/test/LedgerHarness.py b/test/LedgerHarness.py new file mode 100755 index 00000000..0668cfa6 --- /dev/null +++ b/test/LedgerHarness.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python + +import sys +import os +import re + +from subprocess import Popen, PIPE + +class LedgerHarness: + ledger = None + succeeded = 0 + failed = 0 + verify = False + gmalloc = False + + def __init__(self, argv): + if not os.path.isfile(argv[1]): + print "Cannot find ledger at '%s'" % argv[1] + sys.exit(1) + + self.ledger = argv[1] + self.succeeded = 0 + self.failed = 0 + self.verify = '--verify' in argv + self.gmalloc = '--gmalloc' in argv + + def run(self, command, verify=None, gmalloc=None, columns=True): + env = os.environ.copy() + + if (gmalloc is not None and gmalloc) or \ + (gmalloc is None and self.gmalloc): + env['MallocGuardEdges'] = '1' + env['MallocScribble'] = '1' + env['MallocPreScribble'] = '1' + env['MallocCheckHeapStart'] = '100' + env['MallocCheckHeapEach'] = '100' + env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib' + env['MALLOC_PROTECT_BEFORE'] = '1' + env['MALLOC_FILL_SPACE'] = '1' + env['MALLOC_STRICT_SIZE'] = '1' + + if (verify is not None and verify) or \ + (verify is None and self.verify): + insert = ' --verify' + else: + insert = '' + + if columns: + insert += ' --columns=80' + + command = re.sub('\$ledger', + '%s%s --args-only' % (self.ledger, insert), + command) + + return Popen(command, shell=True, close_fds=True, env=env, + stdin=PIPE, stdout=PIPE, stderr=PIPE) + + def read(self, fd): + text = "" + text_data = os.read(fd.fileno(), 8192) + while text_data: + if text_data: + text += text_data + text_data = os.read(fd.fileno(), 8192) + if text_data: + text += text_data + return text + + def readlines(self, fd): + lines = [] + for line in fd.readlines(): + if line == "GuardMalloc: Allocations will be placed on byte boundaries.\n" or \ + line == "GuardMalloc: - Some buffer overruns may not be noticed.\n" or \ + line == "GuardMalloc: - Applications using vector instructions (e.g., SSE or Altivec) may fail.\n" or \ + line == "GuardMalloc: - Applications expecting word-aligned pointers may fail (such as Carbon applications)\n" or \ + line == "GuardMalloc: GuardMalloc version 18\n": + continue + else: + lines.append(line) + return lines + + def wait(self, process, msg='Ledger invocation failed:'): + if process.wait() != 0: + print msg + print process.stderr.read() + self.failure() + return False + return True + + def success(self): + sys.stdout.write(".") + self.succeeded += 1 + + def failure(self): + sys.stdout.write("E") + self.failed += 1 + + def exit(self): + print + if self.succeeded > 0: + print "OK (%d) " % self.succeeded, + if self.failed > 0: + print "FAILED (%d)" % self.failed, + print + + sys.exit(self.failed) + +if __name__ == '__main__': + harness = LedgerHarness(sys.argv) + proc = harness.run('$ledger -f doc/sample.dat reg') + print 'STDOUT:' + print proc.stdout.read() + print 'STDERR:' + print proc.stderr.read() + harness.success() + harness.exit() diff --git a/test/RegressTests.py b/test/RegressTests.py index 2320a83a..e59ecc50 100755 --- a/test/RegressTests.py +++ b/test/RegressTests.py @@ -3,26 +3,20 @@ import sys import os import re -import string import tempfile +from string import join from difflib import unified_diff -from subprocess import Popen, PIPE -ledger = sys.argv[1] -tests = sys.argv[2] +from LedgerHarness import LedgerHarness + +harness = LedgerHarness(sys.argv) +tests = sys.argv[2] -if not os.path.isfile(ledger): - sys.exit(1) if not os.path.isdir(tests) and not os.path.isfile(tests): sys.exit(1) -succeeded = 0 -failed = 0 - def test_regression(test_file): - global succeeded, failed - bug = open(test_file) command = bug.readline() @@ -40,23 +34,14 @@ def test_regression(test_file): if command.startswith("-f - "): use_stdin = True - if re.search('--columns', command): - command = ("%s" % ledger) + " --verify --args-only " + command - else: - command = (("%s" % ledger) + - " --verify --args-only --columns=80 " + command) + command = '$ledger ' + command else: tempdata = tempfile.mkstemp() - os.write(tempdata[0], string.join(data, '')) + os.write(tempdata[0], join(data, '')) os.close(tempdata[0]) - if re.search('--columns', command): - command = (("%s -f \"%s\" " % (ledger, tempdata[1])) + - " --verify --args-only " + command) - else: - command = (("%s -f \"%s\" " % (ledger, tempdata[1])) + - " --verify --args-only --columns=80 " + command) + command = ('$ledger -f "%s" ' % tempdata[1]) + command output = [] while line != ">>>2\n": @@ -74,26 +59,23 @@ def test_regression(test_file): exitcode = int(match.group(1)) - p = Popen(command[:-1], shell=True, - stdin=PIPE, stdout=PIPE, stderr=PIPE, - close_fds=True) + p = harness.run(command, columns=(not re.search('--columns', command))) if use_stdin: - p.stdin.write(string.join(data)) + p.stdin.write(join(data)) p.stdin.close() success = True printed = False index = 0 - for line in unified_diff(output, p.stdout.readlines()): + for line in unified_diff(output, harness.readlines(p.stdout)): index += 1 if index < 3: continue if not printed: if success: print - print "Regression failure in output from %s:" \ - % os.path.basename(test_file) - if success: failed += 1 + print "Regression failure in output from %s:" % \ + os.path.basename(test_file) success = False printed = True print " ", line, @@ -101,32 +83,25 @@ def test_regression(test_file): printed = False index = 0 for line in unified_diff([re.sub('\$FILE', tempdata[1], line) - for line in error], - p.stderr.readlines()): + for line in error], harness.readlines(p.stderr)): index += 1 if index < 3: continue if not printed: if success: print - print "Regression failure in error output from %s:" \ - % os.path.basename(test_file) - if success: failed += 1 + print "Regression failure in error output from %s:" % \ + os.path.basename(test_file) success = False printed = True print " ", line, - if exitcode != p.wait(): - if success: print - if success: failed += 1 - success = False - print "Regression failure in exitcode from %s: %d (expected) != %d" \ - % (os.path.basename(test_file), exitcode, p.returncode) - - if success: - succeeded += 1 - sys.stdout.write(".") + if exitcode == p.wait(): + harness.success() else: - sys.stdout.write("E") + if success: print + print "Regression failure in exitcode from %s: %d (expected) != %d" % \ + (os.path.basename(test_file), exitcode, p.returncode) + harness.failure() if not use_stdin: os.remove(tempdata[1]) @@ -138,11 +113,4 @@ if os.path.isdir(tests): else: test_regression(tests) -print -if succeeded > 0: - print "OK (%d) " % succeeded, -if failed > 0: - print "FAILED (%d)" % failed, -print - -sys.exit(failed) +harness.exit() diff --git a/test/fullcheck.sh b/test/fullcheck.sh index 5763278c..f3c20dd2 100755 --- a/test/fullcheck.sh +++ b/test/fullcheck.sh @@ -11,5 +11,8 @@ export MallocPreScribble=1 export MallocCheckHeapStart=100 export MallocCheckHeapEach=100 export DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib +export MALLOC_PROTECT_BEFORE=1 +export MALLOC_FILL_SPACE=1 +export MALLOC_STRICT_SIZE=1 exec $VALGRIND $@ diff --git a/tools/times.sh b/tools/times.sh new file mode 100755 index 00000000..444da993 --- /dev/null +++ b/tools/times.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +time test/RegressTests.py ./ledger test/regress +time test/RegressTests.py ./ledger test/regress --verify +time test/RegressTests.py ./ledger test/regress --gmalloc +time test/RegressTests.py ./ledger test/regress --verify --gmalloc |