Added LedgerHarness.py, to simplify test creation

5 files changed, 198 insertions, 174 deletions

diff --git a/test/ConfirmTests.py b/test/ConfirmTests.py
index 4b64af7c..0b3d4897 100755
--- a/test/ConfirmTests.py
+++ b/test/ConfirmTests.py
@@ -7,7 +7,13 @@ import sys
 import os
 import re
 
-from subprocess import Popen, PIPE
+from LedgerHarness import LedgerHarness
+
+harness = LedgerHarness(sys.argv)
+tests   = sys.argv[2]
+
+if not os.path.isdir(tests) and not os.path.isfile(tests):
+    sys.exit(1)
 
 commands = [
     "-f '$tests/standard.dat' -O 0ecbb1b15e2cf3e515cc0f8533e5bb0fb2326728",
@@ -19,16 +25,6 @@ commands = [
     "-f '$tests/standard.dat' -G c0226fafdf9e6711ac9121cf263e2d50791859cb"
 ]
 
-ledger    = sys.argv[1]
-tests     = sys.argv[2]
-succeeded = 0
-failed    = 0
-
-if not os.path.isfile(ledger):
-    sys.exit(1)
-if not os.path.isdir(tests) and not os.path.isfile(tests):
-    sys.exit(1)
-
 def clean(num):
     num = re.sub("(\s+|\$|,)","", num)
     m = re.search("([-0-9.]+)", num)
@@ -37,26 +33,25 @@ def clean(num):
     else:
         return float(num)
 
-def confirm_report(args):
+def confirm_report(command):
     index         = 1
     last_line     = ""
     failure       = False
     running_total = 0.0
 
-    p = Popen(re.sub('\$cmd', 'reg', args), shell=True,
-              stdin=PIPE, stdout=PIPE, stderr=PIPE,
-              close_fds=True)
+    p = harness.run(re.sub('\$cmd', 'reg', command))
 
-    for line in p.stdout.readlines():
+    for line in harness.readlines(p.stdout):
         match = re.match("\\s*([-$,0-9.]+)\\s+([-$,0-9.]+)", line[54:])
         if not match:
             continue
+
         value = clean(match.group(1))
         total = clean(match.group(2))
-
         running_total += value
+
         diff = abs(running_total - total)
-        if re.search(' -[VGB] ', args) and diff < 0.015:
+        if re.search(' -[VGB] ', command) and diff < 0.015:
             diff = 0.0
         if diff > 0.001:
             print "DISCREPANCY: %.3f (%.3f - %.3f) at line %d:" % \
@@ -70,16 +65,14 @@ def confirm_report(args):
 
     balance_total = 0.0
 
-    p = Popen(re.sub('\$cmd', 'bal', args), shell=True,
-              stdin=PIPE, stdout=PIPE, stderr=PIPE,
-              close_fds=True)
+    p = harness.run(re.sub('\$cmd', 'bal', command))
 
-    for line in p.stdout.readlines():
+    for line in harness.readlines(p.stdout):
         if line[0] != '-':
             balance_total = clean(line[:20])
 
     diff = abs(balance_total - running_total)
-    if re.search(' -[VGB] ', args) and diff < 0.015:
+    if re.search(' -[VGB] ', command) and diff < 0.015:
         diff = 0.0
     if diff > 0.001:
         print
@@ -91,20 +84,9 @@ def confirm_report(args):
     return not failure
 
 for cmd in commands:
-    if confirm_report("%s --args-only --verify --columns=80 %s" %
-                      (ledger, re.sub('\$tests', tests, cmd))):
-        sys.stdout.write(".")
-        succeeded += 1
+    if confirm_report('$ledger $cmd ' + re.sub('\$tests', tests, cmd)):
+        harness.success()
     else:
-        sys.stdout.write("E")
-        failed += 1
-
-print
-if succeeded > 0:
-    print "OK (%d) " % succeeded,
-if failed > 0:
-    print "FAILED (%d)" % failed,
-print
-
-sys.exit(failed)
+        harness.failure()
 
+harness.exit()
diff --git a/test/GenerateTests.py b/test/GenerateTests.py
index 88c223cd..aa36737d 100755
--- a/test/GenerateTests.py
+++ b/test/GenerateTests.py
@@ -4,68 +4,44 @@
 # final balance is the same as what the balance report shows.
 
 import sys
-import os
-import re
+#import re
 
-from subprocess import Popen, PIPE
 from difflib import ndiff
 
-ledger    = sys.argv[1]
-succeeded = 0
-failed    = 0
-
-if not os.path.isfile(ledger):
-    sys.exit(1)
-
-def normalize(line):
-    match = re.match("((\s*)([A-Za-z]+)?(\s*)([-0-9.]+)(\s*)([A-Za-z]+)?)(  (.+))?$", line)
-    if match:
-        if match.group(3):
-            prefix = match.group(3) + " " + match.group(5)
-            if match.group(8):
-                return prefix + match.group(8)
-            return prefix
-        elif match.group(7):
-            prefix = match.group(7) + " " + match.group(5)
-            if match.group(8):
-                return prefix + match.group(8)
-            return prefix
-    return line
+from LedgerHarness import LedgerHarness
+
+harness = LedgerHarness(sys.argv)
+
+#def normalize(line):
+#    match = re.match("((\s*)([A-Za-z]+)?(\s*)([-0-9.]+)(\s*)([A-Za-z]+)?)(  (.+))?$", line)
+#    if match:
+#        if match.group(3):
+#            prefix = match.group(3) + " " + match.group(5)
+#            if match.group(8):
+#                return prefix + match.group(8)
+#            return prefix
+#        elif match.group(7):
+#            prefix = match.group(7) + " " + match.group(5)
+#            if match.group(8):
+#                return prefix + match.group(8)
+#            return prefix
+#    return line
 
 def generation_test(seed):
-    global succeeded, failed
-
-    p_gen = Popen("%s --args-only --actual --seed=%d generate" % (ledger, seed),
-                  shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE,
-                  close_fds=True)
-    cout = ""
-    cout_data = os.read(p_gen.stdout.fileno(), 8192)
-    while cout_data:
-        if cout_data:
-            cout += cout_data
-        cout_data = os.read(p_gen.stdout.fileno(), 8192)
-    if cout_data:
-        cout += cout_data
-
-    if p_gen.wait() != 0:
-        print "Generation for seed %d failed due to error:" % seed
-        print p_gen.stderr.read()
-        del p_gen
+    p_gen = harness.run('$ledger --seed=%d generate' % seed)
+
+    cout = harness.read(p_gen.stdout)
+
+    if not harness.wait(p_gen, msg=("Generation for seed %d failed:" % seed)):
         return False
-    del p_gen
 
-    p_print = Popen("%s --args-only --actual -f - print" % ledger, shell=True,
-                    stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
+    p_print = harness.run('$ledger --actual -f - print')
     p_print.stdin.write(cout)
     p_print.stdin.close()
     p_print_out = p_print.stdout.read()
 
-    if p_print.wait() != 0:
-        print "Print for seed %d failed due to error:" % seed
-        print p_print.stderr.read()
-        del p_print
+    if not harness.wait(p_print, msg=("Print for seed %d failed:" % seed)):
         return False
-    del p_print
 
     #p_cerr_bal = Popen("%s --args-only -f - bal" % ledger, shell=True,
     #                   stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
@@ -81,34 +57,24 @@ def generation_test(seed):
     #    return False
     #del p_cerr_bal
 
-    p_cout_bal = Popen("%s --args-only -f - bal" % ledger, shell=True,
-                       stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
+    p_cout_bal = harness.run('$ledger -f - bal')
     p_cout_bal.stdin.write(cout)
     p_cout_bal.stdin.close()
 
-    cout_lines = p_cout_bal.stdout.readlines()
-    norm_cout_lines = [normalize(line) for line in cout_lines]
+    cout_lines = harness.readlines(p_cout_bal.stdout)
+    #norm_cout_lines = [normalize(line) for line in cout_lines]
 
-    if p_cout_bal.wait() != 0:
-        print "Stdout balance for seed %d failed due to error:" % seed
-        print p_cout_bal.stderr.read()
-        del p_cout_bal
+    if not harness.wait(p_cout_bal, msg=("Stdout balance for seed %d failed:" % seed)):
         return False
-    del p_cout_bal
 
-    p_print_bal = Popen("%s --args-only -f - bal" % ledger, shell=True,
-                        stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
+    p_print_bal = harness.run('$ledger -f - bal')
     p_print_bal.stdin.write(p_print_out)
     p_print_bal.stdin.close()
 
-    print_lines = p_print_bal.stdout.readlines()
+    print_lines = harness.readlines(p_print_bal.stdout)
 
-    if p_print_bal.wait() != 0:
-        print "Print balance for seed %d failed due to error:" % seed
-        print p_print_bal.stderr.read()
-        del p_print_bal
+    if not harness.wait(p_print_bal, msg=("Print balance for seed %d failed:" % seed)):
         return False
-    del p_print_bal
 
     success = True
     #printed = False
@@ -130,7 +96,6 @@ def generation_test(seed):
         if not printed:
             if success: print
             print "Generation failure in output from seed %d (cout vs. print):" % seed
-            if success: failed += 1
             success = False
             printed = True
         print " ", line
@@ -145,18 +110,8 @@ if len(sys.argv) > 3:
 
 for i in range(beg_range, end_range):
     if generation_test(i):
-        sys.stdout.write(".")
-        succeeded += 1
+        harness.success()
     else:
-        sys.stdout.write("E")
-        failed += 1
-
-print
-if succeeded > 0:
-    print "OK (%d) " % succeeded,
-if failed > 0:
-    print "FAILED (%d)" % failed,
-print
-
-sys.exit(failed)
+        harness.failure()
 
+harness.exit()
diff --git a/test/LedgerHarness.py b/test/LedgerHarness.py
new file mode 100755
index 00000000..0668cfa6
--- /dev/null
+++ b/test/LedgerHarness.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+
+from subprocess import Popen, PIPE
+
+class LedgerHarness:
+    ledger    = None
+    succeeded = 0
+    failed    = 0
+    verify    = False
+    gmalloc   = False
+
+    def __init__(self, argv):
+        if not os.path.isfile(argv[1]):
+            print "Cannot find ledger at '%s'" % argv[1]
+            sys.exit(1)
+
+        self.ledger    = argv[1]
+        self.succeeded = 0
+        self.failed    = 0
+        self.verify    = '--verify' in argv
+        self.gmalloc   = '--gmalloc' in argv
+
+    def run(self, command, verify=None, gmalloc=None, columns=True):
+        env = os.environ.copy()
+
+        if (gmalloc is not None and gmalloc) or \
+           (gmalloc is None and self.gmalloc):
+            env['MallocGuardEdges']      = '1'
+            env['MallocScribble']        = '1'
+            env['MallocPreScribble']     = '1'
+            env['MallocCheckHeapStart']  = '100'
+            env['MallocCheckHeapEach']   = '100'
+            env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib'
+            env['MALLOC_PROTECT_BEFORE'] = '1'
+            env['MALLOC_FILL_SPACE']     = '1'
+            env['MALLOC_STRICT_SIZE']    = '1'
+
+        if (verify is not None and verify) or \
+           (verify is None and self.verify):
+            insert = ' --verify'
+        else:
+            insert = ''
+
+        if columns:
+            insert += ' --columns=80'
+
+        command = re.sub('\$ledger',
+                         '%s%s --args-only' % (self.ledger, insert),
+                         command)
+
+        return Popen(command, shell=True, close_fds=True, env=env,
+                     stdin=PIPE, stdout=PIPE, stderr=PIPE)
+
+    def read(self, fd):
+        text = ""
+        text_data = os.read(fd.fileno(), 8192)
+        while text_data:
+            if text_data:
+                text += text_data
+            text_data = os.read(fd.fileno(), 8192)
+        if text_data:
+            text += text_data
+        return text
+
+    def readlines(self, fd):
+        lines = []
+        for line in fd.readlines():
+            if line == "GuardMalloc: Allocations will be placed on byte boundaries.\n" or \
+               line == "GuardMalloc:  - Some buffer overruns may not be noticed.\n" or \
+               line == "GuardMalloc:  - Applications using vector instructions (e.g., SSE or Altivec) may fail.\n" or \
+               line == "GuardMalloc:  - Applications expecting word-aligned pointers may fail (such as Carbon applications)\n" or \
+               line == "GuardMalloc: GuardMalloc version 18\n":
+                continue
+            else:
+                lines.append(line)
+        return lines
+
+    def wait(self, process, msg='Ledger invocation failed:'):
+        if process.wait() != 0:
+            print msg
+            print process.stderr.read()
+            self.failure()
+            return False
+        return True
+
+    def success(self):
+        sys.stdout.write(".")
+        self.succeeded += 1
+
+    def failure(self):
+        sys.stdout.write("E")
+        self.failed += 1
+
+    def exit(self):
+        print
+        if self.succeeded > 0:
+            print "OK (%d) " % self.succeeded,
+        if self.failed > 0:
+            print "FAILED (%d)" % self.failed,
+        print
+
+        sys.exit(self.failed)
+
+if __name__ == '__main__':
+    harness = LedgerHarness(sys.argv)
+    proc = harness.run('$ledger -f doc/sample.dat reg')
+    print 'STDOUT:'
+    print proc.stdout.read()
+    print 'STDERR:'
+    print proc.stderr.read()
+    harness.success()
+    harness.exit()
diff --git a/test/RegressTests.py b/test/RegressTests.py
index 2320a83a..e59ecc50 100755
--- a/test/RegressTests.py
+++ b/test/RegressTests.py
@@ -3,26 +3,20 @@
 import sys
 import os
 import re
-import string
 import tempfile
 
+from string import join
 from difflib import unified_diff
-from subprocess import Popen, PIPE
 
-ledger = sys.argv[1]
-tests  = sys.argv[2]
+from LedgerHarness import LedgerHarness
+
+harness = LedgerHarness(sys.argv)
+tests   = sys.argv[2]
 
-if not os.path.isfile(ledger):
-    sys.exit(1)
 if not os.path.isdir(tests) and not os.path.isfile(tests):
     sys.exit(1)
 
-succeeded = 0
-failed    = 0
-
 def test_regression(test_file):
-    global succeeded, failed
-
     bug     = open(test_file)
     command = bug.readline()
 
@@ -40,23 +34,14 @@ def test_regression(test_file):
     if command.startswith("-f - "):
         use_stdin = True
 
-        if re.search('--columns', command):
-            command = ("%s" % ledger) + " --verify --args-only " + command
-        else:
-            command = (("%s" % ledger) +
-                       " --verify --args-only --columns=80 " + command)
+        command = '$ledger ' + command
     else:
         tempdata = tempfile.mkstemp()
 
-        os.write(tempdata[0], string.join(data, ''))
+        os.write(tempdata[0], join(data, ''))
         os.close(tempdata[0])
 
-        if re.search('--columns', command):
-            command = (("%s -f \"%s\" " % (ledger, tempdata[1])) +
-                       " --verify --args-only " + command)
-        else:
-            command = (("%s -f \"%s\" " % (ledger, tempdata[1])) +
-                       " --verify --args-only --columns=80 " + command)
+        command = ('$ledger -f "%s" ' % tempdata[1]) + command
 
     output = []
     while line != ">>>2\n":
@@ -74,26 +59,23 @@ def test_regression(test_file):
 
     exitcode = int(match.group(1))
 
-    p = Popen(command[:-1], shell=True,
-              stdin=PIPE, stdout=PIPE, stderr=PIPE,
-              close_fds=True)
+    p = harness.run(command, columns=(not re.search('--columns', command)))
 
     if use_stdin:
-        p.stdin.write(string.join(data))
+        p.stdin.write(join(data))
     p.stdin.close()
 
     success = True
     printed = False
     index   = 0
-    for line in unified_diff(output, p.stdout.readlines()):
+    for line in unified_diff(output, harness.readlines(p.stdout)):
         index += 1
         if index < 3:
             continue
         if not printed:
             if success: print
-            print "Regression failure in output from %s:" \
-                % os.path.basename(test_file)
-            if success: failed += 1
+            print "Regression failure in output from %s:" % \
+                os.path.basename(test_file)
             success = False
             printed = True
         print " ", line,
@@ -101,32 +83,25 @@ def test_regression(test_file):
     printed = False
     index   = 0
     for line in unified_diff([re.sub('\$FILE', tempdata[1], line)
-                              for line in error],
-                             p.stderr.readlines()):
+                              for line in error], harness.readlines(p.stderr)):
         index += 1
         if index < 3:
             continue
         if not printed:
             if success: print
-            print "Regression failure in error output from %s:" \
-                % os.path.basename(test_file)
-            if success: failed += 1
+            print "Regression failure in error output from %s:" % \
+                os.path.basename(test_file)
             success = False
             printed = True
         print " ", line,
 
-    if exitcode != p.wait():
-        if success: print
-        if success: failed += 1
-        success = False
-        print "Regression failure in exitcode from %s: %d (expected) != %d" \
-            % (os.path.basename(test_file), exitcode, p.returncode)
-
-    if success:
-        succeeded += 1
-        sys.stdout.write(".")
+    if exitcode == p.wait():
+        harness.success()
     else:
-        sys.stdout.write("E")
+        if success: print
+        print "Regression failure in exitcode from %s: %d (expected) != %d" % \
+            (os.path.basename(test_file), exitcode, p.returncode)
+        harness.failure()
 
     if not use_stdin:
         os.remove(tempdata[1])
@@ -138,11 +113,4 @@ if os.path.isdir(tests):
 else:
     test_regression(tests)
 
-print
-if succeeded > 0:
-    print "OK (%d) " % succeeded,
-if failed > 0:
-    print "FAILED (%d)" % failed,
-print
-
-sys.exit(failed)
+harness.exit()
diff --git a/test/fullcheck.sh b/test/fullcheck.sh
index 5763278c..f3c20dd2 100755
--- a/test/fullcheck.sh
+++ b/test/fullcheck.sh
@@ -11,5 +11,8 @@ export MallocPreScribble=1
 export MallocCheckHeapStart=100
 export MallocCheckHeapEach=100
 export DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib
+export MALLOC_PROTECT_BEFORE=1
+export MALLOC_FILL_SPACE=1
+export MALLOC_STRICT_SIZE=1
 
 exec $VALGRIND $@