[doc] Add CheckTests to ctest

to check whether all available ledger options documented
and are being tested.

Signed-off-by: Alexis Hildebrandt <afh@surryhill.net>

6 files changed, 156 insertions, 79 deletions

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 44db81fb..37224d40 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -42,10 +42,15 @@ if (PYTHONINTERP_FOUND)
   foreach(TestFile ${${_class}_TESTS})
     get_filename_component(TestFile_Name ${TestFile} NAME_WE)
     add_test(NAME ${_class}Test_${TestFile_Name}
-      COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/DocTests.py
+      COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/${_class}.py
       --ledger $<TARGET_FILE:ledger> --file ${TestFile})
     set_target_properties(check PROPERTIES DEPENDS ${_class}Test_${TestFile_Name})
   endforeach()
+
+  set(_class CheckTests)
+  add_test(NAME ${_class}
+    COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/${_class}.py
+    --ledger $<TARGET_FILE:ledger> --source ${PROJECT_SOURCE_DIR})
 endif()
 
 ### CMakeLists.txt ends here
diff --git a/test/CheckTests.py b/test/CheckTests.py
index 1a364ff4..cb9e431e 100755
--- a/test/CheckTests.py
+++ b/test/CheckTests.py
@@ -1,87 +1,136 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
 
 import sys
 import re
 import os
+import argparse
 
 from os.path import *
 from subprocess import Popen, PIPE
 
-ledger_binary = sys.argv[1]
-source_topdir = sys.argv[2]
-
-documented_options = []
-for line in open(join(source_topdir, 'doc', 'ledger.1')):
-    match = re.match('\.It Fl \\\\-([-A-Za-z]+)', line)
-    if match:
-        option = match.group(1)
-        if option not in documented_options:
-            documented_options.append(option)
-
-pipe   = Popen('%s --debug option.names parse true' % ledger_binary,
-               shell=True, stdout=PIPE, stderr=PIPE)
-errors = 0
-
-untested_options = [
-    'anon',
-    'args-only',
-    'cache',
-    'debug',
-    'download',
-    'file',
-    'force-color',
-    'force-pager',
-    'full-help',
-    'help',
-    'help-calc',
-    'help-comm',
-    'help-disp',
-    'import',
-    'init-file',
-    'no-color',
-    'options',
-    'price-db',
-    'price-exp',
-    'revalued-total',
-    'script',
-    'seed',
-    'trace',
-    'verbose',
-    'verify',
-    'version'
-]
-
-for line in pipe.stderr:
-    match = re.search('\[DEBUG\] Option: (.*)', line)
-    if match:
-        option = match.group(1)
-
-        option = re.sub('_', '-', option)
-        option = re.sub('-$', '', option)
-
-        if option not in untested_options and \
-           not exists(join(source_topdir, 'test', 'baseline',
-                           'opt-%s.test' % option)):
-            print "Baseline test missing for --%s" % option
-            errors += 1
-
-        if option not in documented_options:
-            print "Man page entry missing for --%s" % option
-            errors += 1
-        else:
-            documented_options.remove(option)
-
-known_alternates = [
-    'cost',
-    'first',
-    'import',
-    'last',
-    'leeway',
-    'period-sort'
-]
-
-for option in documented_options:
-    if option not in known_alternates:
-        print "Man page entry for unknown option --%s" % option
-
-sys.exit(errors)
+class CheckTests (object):
+  def __init__(self, args):
+    self.ledger     = os.path.abspath(args.ledger)
+    self.source     = os.path.abspath(args.source)
+
+    self.missing_baseline_tests = set()
+    self.missing_texi_options = set()
+    self.unknown_texi_options = set()
+    self.missing_man_options = set()
+    self.unknown_man_options = set()
+
+    self.untested_options = [
+        'anon',
+        'args-only',
+        'cache',
+        'debug',
+        'download',
+        'file',
+        'force-color',
+        'force-pager',
+        'full-help',
+        'help',
+        'help-calc',
+        'help-comm',
+        'help-disp',
+        'import',
+        'init-file',
+        'no-color',
+        'options',
+        'price-db',
+        'price-exp',
+        'revalued-total',
+        'script',
+        'seed',
+        'trace',
+        'verbose',
+        'verify',
+        'version'
+    ]
+
+    self.known_alternates = [
+        'cost',
+        'first',
+        'import',
+        'last',
+        'leeway',
+        'period-sort'
+    ]
+
+  def find_options(self, pattern, filename):
+    regex = re.compile(pattern)
+    return {match.group(1) for match in {regex.match(line) for line in open(filename)} if match}
+
+  def main(self):
+    man_options = self.find_options('\.It Fl \\\\-([-A-Za-z]+)',
+                                    join(self.source, 'doc', 'ledger.1'))
+
+    texi_options = self.find_options('@item --([-A-Za-z]+).*@c option',
+                                     join(self.source, 'doc', 'ledger3.texi'))
+
+    pipe   = Popen('%s --debug option.names parse true' % self.ledger,
+                   shell=True, stdout=PIPE, stderr=PIPE)
+    regex = re.compile('\[DEBUG\] Option: (.*)')
+    for line in filter(regex.search, [line.decode() for line in pipe.stderr]):
+      match = regex.search(line)
+      option = match.group(1)
+      option = re.sub('_', '-', option)
+      option = re.sub('-$', '', option)
+
+      if option not in self.untested_options and \
+         not exists(join(self.source, 'test', 'baseline',
+                         'opt-%s.test' % option)):
+          self.missing_baseline_tests.add(option)
+
+      if option not in man_options:
+          self.missing_man_options.add(option)
+      else:
+          man_options.remove(option)
+
+      if option not in texi_options:
+          self.missing_texi_options.add(option)
+      else:
+          texi_options.remove(option)
+
+    self.unknown_man_options = [option for option in man_options if option not in self.known_alternates]
+    self.unknown_texi_options = [option for option in texi_options if option not in self.known_alternates]
+
+    sep = "\n  --"
+    if len(self.missing_baseline_tests):
+      print("Missing Baseline test for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_baseline_tests)))))
+    if len(self.missing_man_options):
+      print("Missing man page entries for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_man_options)))))
+    if len(self.missing_texi_options):
+      print("Missing texi entries for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_texi_options)))))
+    if len(self.unknown_man_options):
+      print("Man page entry for unknown options:%s%s" % (sep, sep.join(sorted(list(self.unknown_man_options)))))
+    if len(self.unknown_texi_options):
+      print("Texi entry for unknown option:%s%s" % (sep, sep.join(sorted(list(self.unknown_texi_options)))))
+
+    errors = len(self.missing_baseline_tests) + len(self.missing_man_options) + len(self.missing_baseline_tests)
+    return errors
+
+if __name__ == "__main__":
+  def getargs():
+    parser = argparse.ArgumentParser(prog='CheckTests', description='Check that ledger options are tested and documented', prefix_chars='-')
+    parser.add_argument('-l', '--ledger',
+        dest='ledger',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the ledger executable to test with')
+    parser.add_argument('-s', '--source',
+        dest='source',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the top level ledger source directory')
+    return parser.parse_args()
+
+  args = getargs()
+  script = CheckTests(args)
+  status = script.main()
+  sys.exit(status)
diff --git a/test/GenerateTests.py b/test/GenerateTests.py
index 2e956db0..1a9045a2 100755
--- a/test/GenerateTests.py
+++ b/test/GenerateTests.py
@@ -79,6 +79,8 @@ def generation_test(seed):
     p_cout_bal.stdin.close()
 
     cout_lines = harness.readlines(p_cout_bal.stdout)
+    if len(cout_lines) == 0:
+      return False
     #norm_cout_lines = [normalize(line) for line in cout_lines]
 
     if not harness.wait(p_cout_bal, msg=("Stdout balance for seed %d failed:" % seed)):
@@ -89,6 +91,8 @@ def generation_test(seed):
     p_print_bal.stdin.close()
 
     print_lines = harness.readlines(p_print_bal.stdout)
+    if len(print_lines) == 0:
+      return False
 
     if not harness.wait(p_print_bal, msg=("Print balance for seed %d failed:" % seed)):
         return False
diff --git a/test/RegressTests.py b/test/RegressTests.py
index 01e14191..0fef2127 100755
--- a/test/RegressTests.py
+++ b/test/RegressTests.py
@@ -57,6 +57,9 @@ class RegressFile(object):
         in_error  = False
 
         line = self.fd.readline()
+        if not line:
+          print >>sys.stderr, "WARNING: Empty testfile detected: %s" % (self.filename)
+          return False
         #print "line =", line
         while line:
             if line.startswith("test "):
diff --git a/test/baseline/opt-no-titles.test b/test/baseline/opt-no-titles.test
index e69de29b..40a9b421 100644
--- a/test/baseline/opt-no-titles.test
+++ b/test/baseline/opt-no-titles.test
@@ -0,0 +1,10 @@
+test reg -f test/input/drewr3.dat --no-titles --group-by payee reg food
+11-Jan-02 Grocery Store         Expense:Food:Groceries      $ 65.00      $ 65.00
+11-Jan-19 Grocery Store         Expense:Food:Groceries      $ 44.00     $ 109.00
+10-Dec-20 Organic Co-op         Expense:Food:Groceries      $ 37.50      $ 37.50
+                                Expense:Food:Groceries      $ 37.50      $ 75.00
+                                Expense:Food:Groceries      $ 37.50     $ 112.50
+                                Expense:Food:Groceries      $ 37.50     $ 150.00
+                                Expense:Food:Groceries      $ 37.50     $ 187.50
+                                Expense:Food:Groceries      $ 37.50     $ 225.00
+end test
diff --git a/test/baseline/opt-values.test b/test/baseline/opt-values.test
new file mode 100644
index 00000000..35eeeaf6
--- /dev/null
+++ b/test/baseline/opt-values.test
@@ -0,0 +1,6 @@
+test tags -f test/input/drewr3.dat --values
+hastag: not block
+hastag: true
+nestedtag: true
+nobudget
+end test