Make tests scripts Python 3 compatible

15 files changed, 137 insertions, 81 deletions

diff --git a/test/CheckOptions.py b/test/CheckOptions.py
index e4a1fdc3..dc865532 100755
--- a/test/CheckOptions.py
+++ b/test/CheckOptions.py
@@ -36,6 +36,7 @@ class CheckOptions (object):
 
   def find_functions(self, filename):
     return self.find_pattern(filename, self.function_pattern)
+
   def find_alternates(self):
     command = shlex.split('grep --no-filename OPT_ALT')
     for source_file in ['session', 'report']:
@@ -53,7 +54,7 @@ class CheckOptions (object):
     pipe   = Popen('%s --debug option.names parse true' %
         self.ledger, shell=True, stdout=PIPE, stderr=PIPE)
     regex = re.compile('\[DEBUG\]\s+Option:\s+(.*?)_?$')
-    ledger_options = {match.group(1).replace('_', '-') for match in {regex.search(line.decode()) for line in pipe.stderr} if match}
+    ledger_options = {match.group(1).replace('_', '-') for match in {regex.search(line.decode()) for line in pipe.stderr.readlines()} if match}
     return ledger_options
 
   def ledger_functions(self):
diff --git a/test/ConfirmTests.py b/test/ConfirmTests.py
index dffa74a6..2ecefa01 100755
--- a/test/ConfirmTests.py
+++ b/test/ConfirmTests.py
@@ -1,5 +1,7 @@
 #!/usr/bin/python
 
+from __future__ import print_function
+
 # This script confirms both that the register report "adds up", and that its
 # final balance is the same as what the balance report shows.
 
@@ -56,9 +58,9 @@ def confirm_report(command):
         if re.search(' -[VGB] ', command) and diff < 0.015:
             diff = 0.0
         if diff > 0.001:
-            print "DISCREPANCY: %.3f (%.3f - %.3f) at line %d:" % \
+            print("DISCREPANCY: %.3f (%.3f - %.3f) at line %d:" % \)
                   (running_total - total, running_total, total, index)
-            print line,
+            print(line,)
             running_total = total
             failure = True
 
@@ -77,10 +79,10 @@ def confirm_report(command):
     if re.search(' -[VGB] ', command) and diff < 0.015:
         diff = 0.0
     if diff > 0.001:
-        print
-        print "DISCREPANCY: %.3f (%.3f - %.3f) between register and balance" % \
+        print()
+        print("DISCREPANCY: %.3f (%.3f - %.3f) between register and balance" % \)
                   (balance_total - running_total, balance_total, running_total)
-        print last_line,
+        print(last_line,)
         failure = True
 
     return not failure
diff --git a/test/DocTests.py b/test/DocTests.py
index 52632a7b..ef47a8ab 100755
--- a/test/DocTests.py
+++ b/test/DocTests.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import print_function, unicode_literals
+
 import os
 import re
 import sys
@@ -43,7 +45,10 @@ class DocTests:
     return example
 
   def test_id(self, example):
-    return hashlib.sha1(example.rstrip()).hexdigest()[0:7].upper()
+    example_id = example.rstrip()
+    if sys.version_info.major > 2:
+        example_id = example_id.encode('utf-8')
+    return hashlib.sha1(example_id).hexdigest()[0:7].upper()
 
   def find_examples(self):
     startexample = re.compile(r'^@smallexample\s+@c\s+(%s|%s|%s|%s)(?::([\dA-Fa-f]+|validate))?(?:,(.*))?'
@@ -71,13 +76,13 @@ class DocTests:
         test_end_line = self.current_line
 
         if not test_id:
-          print >> sys.stderr, 'Example', test_kind, 'in line', test_begin_line, 'is missing id.'
+          print('Example', test_kind, 'in line', test_begin_line, 'is missing id.', file=sys.stderr)
           test_id = self.test_id(example)
           if test_kind == self.testin_token:
-            print >> sys.stderr, 'Use', self.test_id(example)
+            print('Use', self.test_id(example), file=sys.stderr)
         elif test_kind == self.testin_token and test_id != self.validate_token and test_id != self.test_id(example):
-          print >> sys.stderr, 'Expected test id', test_id, 'for example' \
-              , test_kind, 'on line', test_begin_line, 'to be', self.test_id(example)
+          print('Expected test id', test_id, 'for example' \
+              , test_kind, 'on line', test_begin_line, 'to be', self.test_id(example), file=sys.stderr)
 
         if test_id == self.validate_token:
           test_id = "Val-" + str(test_begin_line)
@@ -118,7 +123,12 @@ class DocTests:
       else:
         return None
 
-    command = filter(lambda x: x != '\n', shlex.split(command))
+    command_parts = shlex.split(command)
+    if sys.version_info.major == 2:
+      command_parts = map(lambda x: unicode(x, 'utf-8'), command_parts)
+    command = filter(lambda x: x != '\n', command_parts)
+    if sys.version_info.major > 2:
+        command = list(command)
     if command[0] == '$': command.remove('$')
     index = command.index('ledger')
     command[index] = self.ledger
@@ -146,7 +156,7 @@ class DocTests:
       tests = list(set(self.tests).intersection(tests))
       temp = list(set(self.tests).difference(tests))
       if len(temp) > 0:
-        print >> sys.stderr, 'Skipping non-existent examples: %s' % ', '.join(temp)
+        print('Skipping non-existent examples: %s' % ', '.join(temp), file=sys.stderr)
     
     for test_id in tests:
       validation = False
@@ -181,7 +191,7 @@ class DocTests:
             elif os.path.exists(os.path.join(test_input_dir, test_file)):
               command[findex] = os.path.join(test_input_dir, test_file)
         try:
-          convert_idx  = command.index('convert')
+          convert_idx  = command.index(str('convert'))
           convert_file = command[convert_idx+1]
           convert_data = example[self.testfile_token][self.testfile_token]
           if not os.path.exists(convert_file):
@@ -194,37 +204,40 @@ class DocTests:
           verify = subprocess.check_output(command, stderr=subprocess.STDOUT)
           if sys.platform == 'win32':
             verify = verify.replace('\r\n', '\n')
+          if sys.version_info.major > 2:
+              verify = verify.decode('utf-8')
           valid = (output == verify) or (not error and validation)
-        except subprocess.CalledProcessError, e:
+        except subprocess.CalledProcessError as e:
           error = e.output
           valid = False
           failed.add(test_id)
         if valid and test_file_created:
           os.remove(test_file)
         if self.verbose > 0:
-          print test_id, ':', 'Passed' if valid else 'FAILED: {}'.format(error) if error else 'FAILED'
+          print(test_id, ':', 'Passed' if valid else 'FAILED: {}'.format(error) if error else 'FAILED')
         else:
           sys.stdout.write('.' if valid else 'E')
+          sys.stdout.flush()
 
         if not (valid or error):
           failed.add(test_id)
           if self.verbose > 1:
-            print ' '.join(command)
+            print(' '.join(command))
             if not validation:
               for line in unified_diff(output.split('\n'), verify.split('\n'), fromfile='generated', tofile='expected'):
                 print(line)
-              print
+              print()
       else:
         if self.verbose > 0:
-          print test_id, ':', 'Skipped'
+          print(test_id, ':', 'Skipped')
         else:
           sys.stdout.write('X')
 
-    if not self.verbose:
-      print
+    if self.verbose == 0:
+      print()
     if len(failed) > 0:
-      print "\nThe following examples failed:"
-      print " ", "\n  ".join(failed)
+      print("\nThe following examples failed:")
+      print(" ", "\n  ".join(failed))
     return len(failed)
 
   def main(self):
@@ -242,6 +255,7 @@ if __name__ == "__main__":
     parser.add_argument('-v', '--verbose',
         dest='verbose',
         action='count',
+        default=0,
         help='be verbose. Add -vv for more verbosity')
     parser.add_argument('-l', '--ledger',
         dest='ledger',
diff --git a/test/GenerateTests.py b/test/GenerateTests.py
index 1a9045a2..d5851821 100755
--- a/test/GenerateTests.py
+++ b/test/GenerateTests.py
@@ -3,6 +3,8 @@
 # This script confirms both that the register report "adds up", and that its
 # final balance is the same as what the balance report shows.
 
+from __future__ import print_function
+
 import sys
 import re
 
@@ -68,8 +70,8 @@ def generation_test(seed):
     #cerr_lines = [normalize(line) for line in p_cerr_bal.stdout.readlines()]
     #
     #if p_cerr_bal.wait() != 0:
-    #    print "Stderr balance for seed %d failed due to error:" % seed
-    #    print p_cerr_bal.stderr.read()
+    #    print("Stderr balance for seed %d failed due to error:" % seed)
+    #    print(p_cerr_bal.stderr.read())
     #    del p_cerr_bal
     #    return False
     #del p_cerr_bal
@@ -103,23 +105,23 @@ def generation_test(seed):
     #    if line[:2] == "  ":
     #        continue
     #    if not printed:
-    #        if success: print
-    #        print "Generation failure in output from seed %d (cerr vs. cout):" % seed
+    #        if success: print()
+    #        print("Generation failure in output from seed %d (cerr vs. cout):" % seed)
     #        if success: failed += 1
     #        success = False
     #        printed = True
-    #    print " ", line
+    #    print(" ", line)
 
     printed = False
     for line in ndiff(cout_lines, print_lines, charjunk=None):
         if line[:2] == "  ":
             continue
         if not printed:
-            if success: print
-            print "Generation failure in output from seed %d (cout vs. print):" % seed
+            if success: print()
+            print("Generation failure in output from seed %d (cout vs. print):" % seed)
             success = False
             printed = True
-        print " ", line
+        print(" ", line)
 
     return success
 
diff --git a/test/LedgerHarness.py b/test/LedgerHarness.py
index 63d6a0ec..2373ea32 100755
--- a/test/LedgerHarness.py
+++ b/test/LedgerHarness.py
@@ -1,4 +1,7 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
 
 import sys
 import os
@@ -6,8 +9,11 @@ import re
 
 from subprocess import Popen, PIPE
 
-import copy_reg
 import types
+if sys.version_info.major == 2:
+    import copy_reg as copyreg
+else:
+    import copyreg
 
 def _pickle_method(method):
     func_name = method.im_func.__name__
@@ -25,7 +31,7 @@ def _unpickle_method(func_name, obj, cls):
             break
     return func.__get__(obj, cls)
 
-copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)
+copyreg.pickle(types.MethodType, _pickle_method, _unpickle_method)
 
 class LedgerHarness:
     ledger     = None
@@ -38,10 +44,10 @@ class LedgerHarness:
 
     def __init__(self, argv):
         if not os.path.isfile(argv[1]):
-            print "Cannot find ledger at '%s'" % argv[1]
+            print("Cannot find ledger at '%s'" % argv[1])
             sys.exit(1)
         if not os.path.isdir(argv[2]):
-            print "Cannot find source path at '%s'" % argv[2]
+            print("Cannot find source path at '%s'" % argv[2])
             sys.exit(1)
 
         self.ledger     = os.path.abspath(argv[1])
@@ -109,14 +115,18 @@ class LedgerHarness:
     def readlines(self, fd):
         lines = []
         for line in fd.readlines():
-            if not line.startswith("GuardMalloc"):
+            if sys.version_info.major == 2:
+                line = unicode(line, 'utf-8')
+            else:
+                line = line.decode('utf-8')
+            if not line.startswith('GuardMalloc'):
                 lines.append(line)
         return lines
 
     def wait(self, process, msg='Ledger invocation failed:'):
         if process.wait() != 0:
-            print msg
-            print process.stderr.read()
+            print(msg)
+            print(process.stderr.read())
             self.failure()
             return False
         return True
@@ -134,21 +144,21 @@ class LedgerHarness:
         self.failed += 1
 
     def exit(self):
-        print
+        print()
         if self.succeeded > 0:
-            print "OK (%d) " % self.succeeded,
+            print("OK (%d) " % self.succeeded,)
         if self.failed > 0:
-            print "FAILED (%d)" % self.failed,
-        print
+            print("FAILED (%d)" % self.failed,)
+        print()
 
         sys.exit(self.failed)
 
 if __name__ == '__main__':
     harness = LedgerHarness(sys.argv)
     proc = harness.run('$ledger -f doc/sample.dat reg')
-    print 'STDOUT:'
-    print proc.stdout.read()
-    print 'STDERR:'
-    print proc.stderr.read()
+    print('STDOUT:')
+    print(proc.stdout.read())
+    print('STDERR:')
+    print(proc.stderr.read())
     harness.success()
     harness.exit()
diff --git a/test/RegressTests.py b/test/RegressTests.py
index 58869edc..9d8a7af9 100755
--- a/test/RegressTests.py
+++ b/test/RegressTests.py
@@ -1,4 +1,7 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
 
 import sys
 import os
@@ -12,7 +15,6 @@ try:
 except:
     pass
 
-from string import join
 from difflib import unified_diff
 
 from LedgerHarness import LedgerHarness
@@ -57,7 +59,8 @@ class RegressFile(object):
         in_error  = False
 
         line = self.fd.readline()
-        #print "line =", line
+        if sys.version_info.major == 2 and type(line) is str:
+            line = unicode(line, 'utf-8')
         while line:
             if line.startswith("test "):
                 command = line[5:]
@@ -86,15 +89,17 @@ class RegressFile(object):
                         test['output'].append(self.transform_line(line))
 
             line = self.fd.readline()
-            #print "line =", line
+            if sys.version_info.major == 2 and type(line) is str:
+               line = unicode(line, 'utf-8')
+            #print("line =", line)
 
         return test['command'] and test
 
     def notify_user(self, msg, test):
-        print msg
-        print "--"
-        print self.transform_line(test['command']),
-        print "--"
+        print(msg)
+        print("--")
+        print(self.transform_line(test['command']),)
+        print("--")
 
     def run_test(self, test):
         use_stdin = False
@@ -120,7 +125,10 @@ class RegressFile(object):
         if use_stdin:
             fd = open(self.filename)
             try:
-                p.stdin.write(fd.read())
+                stdin = fd.read()
+                if sys.version_info.major > 2:
+                    stdin = stdin.encode('utf-8')
+                p.stdin.write(stdin)
             finally:
                 fd.close()
         p.stdin.close()
@@ -147,7 +155,9 @@ class RegressFile(object):
                     self.notify_user("FAILURE in output from %s:" % self.filename, test)
                     success = False
                     printed = True
-                print " ", line,
+                if sys.version_info.major == 2 and type(line) is str:
+                    line = unicode(line, 'utf-8')
+                print(' ', line,)
 
         printed = False
         index   = 0
@@ -168,15 +178,15 @@ class RegressFile(object):
                                      % self.filename, test)
                     success = False
                     printed = True
-                print " ", line,
+                print(" ", line,)
 
         if test['exitcode'] is None or test['exitcode'] == p.wait():
             if success:
                 harness.success()
             else:
                 harness.failure(os.path.basename(self.filename))
-                print "STDERR:"
-                print p.stderr.read()
+                print("STDERR:")
+                print(p.stderr.read())
         else:
             if success: print
             if test['exitcode']:
@@ -187,7 +197,7 @@ class RegressFile(object):
 
     def run_tests(self):
         if os.path.getsize(self.filename) == 0:
-          print >>sys.stderr, "WARNING: Empty testfile detected: %s" % (self.filename)
+          print("WARNING: Empty testfile detected: %s" % (self.filename), file=sys.stderr)
           harness.failure(os.path.basename(self.filename))
           return False
         test = self.read_test()
diff --git a/test/baseline/feat-option_py.test b/test/baseline/feat-option_py.test
index 1b2a0c79..77a75660 100644
--- a/test/baseline/feat-option_py.test
+++ b/test/baseline/feat-option_py.test
@@ -1,9 +1,9 @@
 python
     def option_pyfirst(context):
-        print "In --pyfirst (from %s)" % context
+        print("In --pyfirst (from %s)" % context)
 
     def option_pysecond(context, val):
-        print "In --pysecond=%s (from %s)" % (val, context)
+        print("In --pysecond=%s (from %s)" % (val, context))
 
 --pyfirst
 --pysecond Hey
diff --git a/test/baseline/feat-value_py.test b/test/baseline/feat-value_py.test
index 5efe315d..2f351d2a 100644
--- a/test/baseline/feat-value_py.test
+++ b/test/baseline/feat-value_py.test
@@ -1,6 +1,7 @@
 python
+    from __future__ import print_function
     def print_type(val):
-        print type(val), val
+        print(type(val), val)
 
 eval print_type(true)
 eval print_type([2010/08/10])
diff --git a/test/regress/4D9288AE.py b/test/regress/4D9288AE.py
index 4f9c9ba9..20ba566b 100644
--- a/test/regress/4D9288AE.py
+++ b/test/regress/4D9288AE.py
@@ -1,4 +1,6 @@
+from __future__ import print_function
+
 import ledger
 
 for post in ledger.read_journal("test/regress/4D9288AE.dat").query("^expenses:"):
-    print post.cost
+    print(post.cost)
diff --git a/test/regress/78AB4B87.py b/test/regress/78AB4B87.py
index fed95b54..affc3496 100644
--- a/test/regress/78AB4B87.py
+++ b/test/regress/78AB4B87.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import ledger
 
 eur = ledger.commodities.find_or_create('EUR')
@@ -7,8 +9,8 @@ total_gbp = ledger.Amount("0.00 GBP")
 total = ledger.Amount("0.00 EUR")
 
 for post in ledger.read_journal("test/regress/78AB4B87.dat").query("^income:"):
-    print post.amount
-    print post.amount.commodity
+    print(post.amount)
+    print(post.amount.commodity)
     if post.amount.commodity == "EUR":
         total_eur += post.amount
     elif post.amount.commodity == "GBP":
@@ -16,12 +18,12 @@ for post in ledger.read_journal("test/regress/78AB4B87.dat").query("^income:"):
 
     a = post.amount.value(eur)
     if a:
-        print "Total is presently: (%s)" % total
-        print "Converted to EUR:   (%s)" % a
+        print("Total is presently: (%s)" % total)
+        print("Converted to EUR:   (%s)" % a)
         total += a
-        print "Total is now:       (%s)" % total
+        print("Total is now:       (%s)" % total)
     else:
-        print "Cannot convert '%s'" % post.amount
-    print
+        print("Cannot convert '%s'" % post.amount)
+    print()
 
-print total
+print(total)
diff --git a/test/regress/9188F587.py b/test/regress/9188F587.py
index 50195252..9095d58c 100644
--- a/test/regress/9188F587.py
+++ b/test/regress/9188F587.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import ledger
 
 eur = ledger.commodities.find_or_create('EUR')
@@ -7,8 +9,8 @@ total_gbp = ledger.Amount("0.00 GBP")
 total = ledger.Amount("0.00 EUR")
 
 for post in ledger.read_journal("test/regress/78AB4B87.dat").query("^income:"):
-    print post.amount
-    print post.amount.commodity
+    print(post.amount)
+    print(post.amount.commodity)
     if post.amount.commodity == "EUR":
         total_eur += post.amount
     elif post.amount.commodity == "GBP":
@@ -16,12 +18,12 @@ for post in ledger.read_journal("test/regress/78AB4B87.dat").query("^income:"):
 
     a = post.amount.value(eur, post.date)
     if a:
-        print "Total is presently: (%s)" % total
-        print "Converted to EUR:   (%s)" % a
+        print("Total is presently: (%s)" % total)
+        print("Converted to EUR:   (%s)" % a)
         total += a
-        print "Total is now:       (%s)" % total
+        print("Total is now:       (%s)" % total)
     else:
-        print "Cannot convert '%s'" % post.amount
-    print
+        print("Cannot convert '%s'" % post.amount)
+    print()
 
-print total
+print(total)
diff --git a/test/regress/B21BF389.py b/test/regress/B21BF389.py
index 707ce340..a49c498c 100644
--- a/test/regress/B21BF389.py
+++ b/test/regress/B21BF389.py
@@ -3,7 +3,11 @@
 
 from __future__ import print_function, unicode_literals
 
+import sys
 import ledger
 
 for post in ledger.read_journal(__file__.replace(".py", "_py.test")).query("income"):
-  print(unicode(post.tag("Reference")))
+  reference = post.tag("Reference")
+  if sys.version_info.major == 2:
+      reference = unicode(reference)
+  print(reference)
diff --git a/test/regress/BF3C1F82-2.test b/test/regress/BF3C1F82-2.test
index 94d59e4d..a7808504 100644
--- a/test/regress/BF3C1F82-2.test
+++ b/test/regress/BF3C1F82-2.test
@@ -1,10 +1,12 @@
 ; Check that include directives are relative for "-f /dev/stdin"
 include non-existent-ledger-file-BF3C1F82
+
 test -f - reg -> 1
 __ERROR__
 While parsing file "", line 2:
 Error: File to include was not found: "./non-existent-ledger-file-BF3C1F82"
 end test
+
 test -f /dev/stdin reg -> 1
 __ERROR__
 While parsing file "", line 2:
diff --git a/test/regress/BF3C1F82.test b/test/regress/BF3C1F82.test
index 9d4937bb..830a43fc 100644
--- a/test/regress/BF3C1F82.test
+++ b/test/regress/BF3C1F82.test
@@ -3,6 +3,7 @@
 2012/02/30 * Test
   a  1
   b
+
 test -f - reg -> 1
 __ERROR__
 While parsing file "", line 3:
@@ -10,6 +11,7 @@ While parsing transaction:
 <no source context>
 Error: Day of month is not valid for year
 end test
+
 test -f /dev/stdin reg -> 1
 __ERROR__
 While parsing file "", line 3:
diff --git a/test/regress/xact_code.py b/test/regress/xact_code.py
index 64abb17d..a697e156 100644
--- a/test/regress/xact_code.py
+++ b/test/regress/xact_code.py
@@ -1,4 +1,6 @@
+from __future__ import print_function
+
 import ledger
 
 for post in ledger.read_journal('test/regress/xact_code.dat').query('expenses'):
-  print post.xact.code
+  print(post.xact.code)