[tests] Refactor CheckTests.py into custom scripts

7 files changed, 244 insertions, 141 deletions

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 37224d40..6eb9f956 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -47,10 +47,12 @@ if (PYTHONINTERP_FOUND)
     set_target_properties(check PROPERTIES DEPENDS ${_class}Test_${TestFile_Name})
   endforeach()
 
-  set(_class CheckTests)
-  add_test(NAME ${_class}
-    COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/${_class}.py
-    --ledger $<TARGET_FILE:ledger> --source ${PROJECT_SOURCE_DIR})
+  list(APPEND CheckOptions CheckManpage CheckTexinfo CheckBaselineTests)
+  foreach(_class ${CheckOptions})
+    add_test(NAME ${_class}
+      COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/${_class}.py
+      --ledger $<TARGET_FILE:ledger> --source ${PROJECT_SOURCE_DIR})
+  endforeach()
 endif()
 
 ### CMakeLists.txt ends here
diff --git a/test/CheckBaselineTests.py b/test/CheckBaselineTests.py
new file mode 100755
index 00000000..404c12db
--- /dev/null
+++ b/test/CheckBaselineTests.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+import sys
+import re
+import os
+import argparse
+
+from os.path import *
+from subprocess import Popen, PIPE
+
+from CheckOptions import CheckOptions
+
+class CheckBaselineTests (CheckOptions):
+  def __init__(self, args):
+    CheckOptions.__init__(self, args)
+    self.missing_baseline_tests = set()
+
+  def main(self):
+    for option in self.ledger_options():
+      if option in self.untested_options: continue
+      baseline_testpath = join(self.source, 'test', 'baseline', 'opt-%s.test' % option)
+      if exists(baseline_testpath) and getsize(baseline_testpath) > 0: continue
+      self.missing_baseline_tests.add(option)
+
+    if len(self.missing_baseline_tests):
+      print("Missing Baseline test for:%s%s\n" % (self.sep, self.sep.join(sorted(list(self.missing_baseline_tests)))))
+
+    errors = len(self.missing_baseline_tests)
+    return errors
+
+if __name__ == "__main__":
+  def getargs():
+    parser = argparse.ArgumentParser(prog='CheckBaselineTests',
+            description='Check that ledger options are tested')
+    parser.add_argument('-l', '--ledger',
+        dest='ledger',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the ledger executable to test with')
+    parser.add_argument('-s', '--source',
+        dest='source',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the top level ledger source directory')
+    return parser.parse_args()
+
+  args = getargs()
+  script = CheckBaselineTests(args)
+  status = script.main()
+  sys.exit(status)
diff --git a/test/CheckManpage.py b/test/CheckManpage.py
new file mode 100755
index 00000000..6b0f2476
--- /dev/null
+++ b/test/CheckManpage.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+import sys
+import re
+import os
+import argparse
+
+from os.path import *
+from subprocess import Popen, PIPE
+
+from CheckOptions import CheckOptions
+
+class CheckManpage (CheckOptions):
+  def __init__(self, args):
+    CheckOptions.__init__(self, args)
+    self.option_pattern = '\.It Fl \\\\-([-A-Za-z]+)'
+    self.source_file = join(self.source, 'doc', 'ledger.1')
+    self.source_type = 'manpage'
+
+if __name__ == "__main__":
+  def getargs():
+    parser = argparse.ArgumentParser(prog='CheckManpage',
+            description='Check that ledger options are documented in the manpage')
+    parser.add_argument('-l', '--ledger',
+        dest='ledger',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the ledger executable to test with')
+    parser.add_argument('-s', '--source',
+        dest='source',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the top level ledger source directory')
+    return parser.parse_args()
+
+  args = getargs()
+  script = CheckManpage(args)
+  status = script.main()
+  sys.exit(status)
diff --git a/test/CheckOptions.py b/test/CheckOptions.py
new file mode 100755
index 00000000..d9565600
--- /dev/null
+++ b/test/CheckOptions.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+import sys
+import re
+import os
+import argparse
+
+from os.path import *
+from subprocess import Popen, PIPE
+
+class CheckOptions (object):
+  def __init__(self, args):
+    self.option_pattern = None
+    self.source_file = None
+    self.sep = "\n  --"
+
+    self.ledger     = os.path.abspath(args.ledger)
+    self.source     = os.path.abspath(args.source)
+
+    self.missing_baseline_tests = set()
+    self.missing_options = set()
+    self.unknown_options = set()
+
+    self.untested_options = [
+        'anon',
+        'args-only',
+        'cache',
+        'debug',
+        'download',
+        'file',
+        'force-color',
+        'force-pager',
+        'full-help',
+        'help',
+        'help-calc',
+        'help-comm',
+        'help-disp',
+        'import',
+        'init-file',
+        'no-color',
+        'options',
+        'price-db',
+        'price-exp',
+        'revalued-total',
+        'script',
+        'seed',
+        'trace',
+        'verbose',
+        'verify',
+        'version'
+    ]
+
+    self.known_alternates = [
+        'cost',
+        'first',
+        'import',
+        'last',
+        'leeway',
+        'period-sort'
+    ]
+
+  def find_options(self, filename):
+    regex = re.compile(self.option_pattern)
+    return {match.group(1) for match in {regex.match(line) for line in open(filename)} if match}
+
+  def ledger_options(self):
+    pipe   = Popen('%s --debug option.names parse true' %
+        self.ledger, shell=True, stdout=PIPE, stderr=PIPE)
+    regex = re.compile('\[DEBUG\]\s+Option:\s+(.*?)_?$')
+    ledger_options = {match.group(1).replace('_', '-') for match in {regex.search(line.decode()) for line in pipe.stderr} if match}
+    return ledger_options
+
+  def main(self):
+    options = self.find_options(self.source_file)
+
+    for option in self.ledger_options():
+      if option not in options:
+          self.missing_options.add(option)
+      else:
+          options.remove(option)
+
+    self.unknown_options = {option for option in options if option not in self.known_alternates}
+
+    if len(self.missing_options):
+      print("Missing %s entries for:%s%s\n" % (self.source_type, self.sep, self.sep.join(sorted(list(self.missing_options)))))
+    if len(self.unknown_options):
+      print("%s entry for unknown options:%s%s" % (self.source_type, self.sep, self.sep.join(sorted(list(self.unknown_options)))))
+
+    errors = len(self.missing_options) + len(self.unknown_options)
+    return errors
diff --git a/test/CheckTests.py b/test/CheckTests.py
deleted file mode 100755
index cb9e431e..00000000
--- a/test/CheckTests.py
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from __future__ import print_function
-
-import sys
-import re
-import os
-import argparse
-
-from os.path import *
-from subprocess import Popen, PIPE
-
-class CheckTests (object):
-  def __init__(self, args):
-    self.ledger     = os.path.abspath(args.ledger)
-    self.source     = os.path.abspath(args.source)
-
-    self.missing_baseline_tests = set()
-    self.missing_texi_options = set()
-    self.unknown_texi_options = set()
-    self.missing_man_options = set()
-    self.unknown_man_options = set()
-
-    self.untested_options = [
-        'anon',
-        'args-only',
-        'cache',
-        'debug',
-        'download',
-        'file',
-        'force-color',
-        'force-pager',
-        'full-help',
-        'help',
-        'help-calc',
-        'help-comm',
-        'help-disp',
-        'import',
-        'init-file',
-        'no-color',
-        'options',
-        'price-db',
-        'price-exp',
-        'revalued-total',
-        'script',
-        'seed',
-        'trace',
-        'verbose',
-        'verify',
-        'version'
-    ]
-
-    self.known_alternates = [
-        'cost',
-        'first',
-        'import',
-        'last',
-        'leeway',
-        'period-sort'
-    ]
-
-  def find_options(self, pattern, filename):
-    regex = re.compile(pattern)
-    return {match.group(1) for match in {regex.match(line) for line in open(filename)} if match}
-
-  def main(self):
-    man_options = self.find_options('\.It Fl \\\\-([-A-Za-z]+)',
-                                    join(self.source, 'doc', 'ledger.1'))
-
-    texi_options = self.find_options('@item --([-A-Za-z]+).*@c option',
-                                     join(self.source, 'doc', 'ledger3.texi'))
-
-    pipe   = Popen('%s --debug option.names parse true' % self.ledger,
-                   shell=True, stdout=PIPE, stderr=PIPE)
-    regex = re.compile('\[DEBUG\] Option: (.*)')
-    for line in filter(regex.search, [line.decode() for line in pipe.stderr]):
-      match = regex.search(line)
-      option = match.group(1)
-      option = re.sub('_', '-', option)
-      option = re.sub('-$', '', option)
-
-      if option not in self.untested_options and \
-         not exists(join(self.source, 'test', 'baseline',
-                         'opt-%s.test' % option)):
-          self.missing_baseline_tests.add(option)
-
-      if option not in man_options:
-          self.missing_man_options.add(option)
-      else:
-          man_options.remove(option)
-
-      if option not in texi_options:
-          self.missing_texi_options.add(option)
-      else:
-          texi_options.remove(option)
-
-    self.unknown_man_options = [option for option in man_options if option not in self.known_alternates]
-    self.unknown_texi_options = [option for option in texi_options if option not in self.known_alternates]
-
-    sep = "\n  --"
-    if len(self.missing_baseline_tests):
-      print("Missing Baseline test for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_baseline_tests)))))
-    if len(self.missing_man_options):
-      print("Missing man page entries for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_man_options)))))
-    if len(self.missing_texi_options):
-      print("Missing texi entries for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_texi_options)))))
-    if len(self.unknown_man_options):
-      print("Man page entry for unknown options:%s%s" % (sep, sep.join(sorted(list(self.unknown_man_options)))))
-    if len(self.unknown_texi_options):
-      print("Texi entry for unknown option:%s%s" % (sep, sep.join(sorted(list(self.unknown_texi_options)))))
-
-    errors = len(self.missing_baseline_tests) + len(self.missing_man_options) + len(self.missing_baseline_tests)
-    return errors
-
-if __name__ == "__main__":
-  def getargs():
-    parser = argparse.ArgumentParser(prog='CheckTests', description='Check that ledger options are tested and documented', prefix_chars='-')
-    parser.add_argument('-l', '--ledger',
-        dest='ledger',
-        type=str,
-        action='store',
-        required=True,
-        help='the path to the ledger executable to test with')
-    parser.add_argument('-s', '--source',
-        dest='source',
-        type=str,
-        action='store',
-        required=True,
-        help='the path to the top level ledger source directory')
-    return parser.parse_args()
-
-  args = getargs()
-  script = CheckTests(args)
-  status = script.main()
-  sys.exit(status)
diff --git a/test/CheckTexinfo.py b/test/CheckTexinfo.py
new file mode 100755
index 00000000..c289da68
--- /dev/null
+++ b/test/CheckTexinfo.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+import sys
+import re
+import os
+import argparse
+
+from os.path import *
+from subprocess import Popen, PIPE
+
+from CheckOptions import CheckOptions
+
+class CheckTexinfo (CheckOptions):
+  def __init__(self, args):
+    CheckOptions.__init__(self, args)
+    self.option_pattern = '@item --([-A-Za-z]+).*@c option'
+    self.source_file = join(self.source, 'doc', 'ledger3.texi')
+    self.source_type = 'texinfo'
+
+if __name__ == "__main__":
+  def getargs():
+    parser = argparse.ArgumentParser(prog='CheckTexinfo',
+            description='Check that ledger options are documented in the texinfo manual')
+    parser.add_argument('-l', '--ledger',
+        dest='ledger',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the ledger executable to test with')
+    parser.add_argument('-s', '--source',
+        dest='source',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the top level ledger source directory')
+    return parser.parse_args()
+
+  args = getargs()
+  script = CheckTexinfo(args)
+  status = script.main()
+  sys.exit(status)
diff --git a/test/DocTests.py b/test/DocTests.py
index ea32608e..e362b980 100755
--- a/test/DocTests.py
+++ b/test/DocTests.py
@@ -221,7 +221,8 @@ class DocTests:
 
 if __name__ == "__main__":
   def getargs():
-    parser = argparse.ArgumentParser(prog='DocTests', description='Test ledger examples from the documentation', prefix_chars='-')
+    parser = argparse.ArgumentParser(prog='DocTests',
+            description='Test and validate ledger examples from the texinfo manual')
     parser.add_argument('-v', '--verbose',
         dest='verbose',
         action='count',