Make test harness more Windows compatible

Windows doesn't seem to use UTF-8 by default, so we can specify
encoding='utf-8'. Also, backslashes are confusing to regex parsers.

3 files changed, 18 insertions, 17 deletions

diff --git a/test/DocTests.py b/test/DocTests.py
index ef47a8ab..4814091c 100755
--- a/test/DocTests.py
+++ b/test/DocTests.py
@@ -2,11 +2,13 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import print_function, unicode_literals
+from io import open
 
 import os
 import re
 import sys
 import shlex
+import locale
 import hashlib
 import argparse
 import subprocess
@@ -45,9 +47,7 @@ class DocTests:
     return example
 
   def test_id(self, example):
-    example_id = example.rstrip()
-    if sys.version_info.major > 2:
-        example_id = example_id.encode('utf-8')
+    example_id = example.rstrip().encode('utf-8')
     return hashlib.sha1(example_id).hexdigest()[0:7].upper()
 
   def find_examples(self):
@@ -123,9 +123,9 @@ class DocTests:
       else:
         return None
 
-    command_parts = shlex.split(command)
     if sys.version_info.major == 2:
-      command_parts = map(lambda x: unicode(x, 'utf-8'), command_parts)
+      command = command.encode(locale.getpreferredencoding())
+    command_parts = shlex.split(command)
     command = filter(lambda x: x != '\n', command_parts)
     if sys.version_info.major > 2:
         command = list(command)
@@ -157,7 +157,7 @@ class DocTests:
       temp = list(set(self.tests).difference(tests))
       if len(temp) > 0:
         print('Skipping non-existent examples: %s' % ', '.join(temp), file=sys.stderr)
-    
+
     for test_id in tests:
       validation = False
       if self.validate_dat_token in self.examples[test_id] or self.validate_cmd_token in self.examples[test_id]:
@@ -186,7 +186,7 @@ class DocTests:
           if not os.path.exists(test_file):
             if input:
               test_file_created = True
-              with open(test_file, 'w') as f:
+              with open(test_file, 'w', encoding='utf-8') as f:
                 f.write(input)
             elif os.path.exists(os.path.join(test_input_dir, test_file)):
               command[findex] = os.path.join(test_input_dir, test_file)
@@ -195,17 +195,16 @@ class DocTests:
           convert_file = command[convert_idx+1]
           convert_data = example[self.testfile_token][self.testfile_token]
           if not os.path.exists(convert_file):
-              with open(convert_file, 'w') as f:
+              with open(convert_file, 'w', encoding='utf-8') as f:
                 f.write(convert_data)
         except ValueError:
          pass
         error = None
         try:
           verify = subprocess.check_output(command, stderr=subprocess.STDOUT)
+          verify = verify.decode('utf-8')
           if sys.platform == 'win32':
             verify = verify.replace('\r\n', '\n')
-          if sys.version_info.major > 2:
-              verify = verify.decode('utf-8')
           valid = (output == verify) or (not error and validation)
         except subprocess.CalledProcessError as e:
           error = e.output
@@ -241,7 +240,7 @@ class DocTests:
     return len(failed)
 
   def main(self):
-    self.file = open(self.sourcepath)
+    self.file = open(self.sourcepath, encoding='utf-8')
     self.current_line = 0
     self.find_examples()
     failed_examples = self.test_examples()
diff --git a/test/LedgerHarness.py b/test/LedgerHarness.py
index 2373ea32..0b4f174d 100755
--- a/test/LedgerHarness.py
+++ b/test/LedgerHarness.py
@@ -82,8 +82,8 @@ class LedgerHarness:
         if columns:
             insert += ' --columns=80'
 
-        command = re.sub('\$ledger', '%s%s %s' % \
-                         (self.ledger, insert, '--args-only'), command)
+        command = command.replace('$ledger', '"%s"%s %s' % \
+                         (self.ledger, insert, '--args-only'))
 
         valgrind = '/usr/bin/valgrind'
         if not os.path.isfile(valgrind):
diff --git a/test/RegressTests.py b/test/RegressTests.py
index 9d8a7af9..7395a4c1 100755
--- a/test/RegressTests.py
+++ b/test/RegressTests.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import print_function, unicode_literals
+from io import open
 
 import sys
 import os
@@ -40,11 +41,11 @@ if not os.path.isdir(tests) and not os.path.isfile(tests):
 class RegressFile(object):
     def __init__(self, filename):
         self.filename = filename
-        self.fd = open(self.filename)
+        self.fd = open(self.filename, encoding='utf-8')
 
     def transform_line(self, line):
-        line = re.sub('\$sourcepath', harness.sourcepath, line)
-        line = re.sub('\$FILE', os.path.abspath(self.filename), line)
+        line = line.replace('$sourcepath', harness.sourcepath)
+        line = line.replace('$FILE', os.path.abspath(self.filename))
         return line
 
     def read_test(self):
@@ -123,7 +124,7 @@ class RegressFile(object):
                         columns=(not re.search('--columns', test['command'])))
 
         if use_stdin:
-            fd = open(self.filename)
+            fd = open(self.filename, encoding='utf-8')
             try:
                 stdin = fd.read()
                 if sys.version_info.major > 2:
@@ -168,6 +169,7 @@ class RegressFile(object):
             if sys.platform == 'win32':
                 process_error = [l.replace('\r\n', '\n').replace('\\', '/')
                                  for l in process_error]
+                test['error'] = [l.replace('\\', '/') for l in test['error']]
             for line in unified_diff(test['error'], process_error):
                 index += 1
                 if index < 3: