Check examples in documentation when running tests

The DocTests.py script will parse a given texinfo file for specially
marked examples, run the ledger command from the example, and check
the result against the example output from the documentation.

3 files changed, 203 insertions, 23 deletions

diff --git a/doc/ledger3.texi b/doc/ledger3.texi
index c85f8446..acf58698 100644
--- a/doc/ledger3.texi
+++ b/doc/ledger3.texi
@@ -346,13 +346,13 @@ for each.
 
 To find the balances of all of your accounts, run this command:
 
-@smallexample
+@smallexample @c command:1071890
 $ ledger -f drewr3.dat balance
 @end smallexample
 
 Ledger will generate:
 
-@smallexample
+@smallexample @c output:1071890
          $ -3,804.00  Assets
           $ 1,396.00    Checking
              $ 30.00      Business
@@ -381,8 +381,11 @@ pare this down to show only the accounts you want.
 
 A more useful report is to show only your Assets and Liabilities:
 
-@smallexample
+@smallexample @c command:5BF4D8E
 $ ledger -f drewr3.dat balance Assets Liabilities
+@end smallexample
+
+@smallexample @c output:5BF4D8E
          $ -3,804.00  Assets
           $ 1,396.00    Checking
              $ 30.00      Business
@@ -402,16 +405,16 @@ $ ledger -f drewr3.dat balance Assets Liabilities
 
 To show all transactions and a running total:
 
-@smallexample
+@smallexample @c command:66E3A2C
 $ ledger -f drewr3.dat register
 @end smallexample
 
 @noindent
 Ledger will generate:
 
-@smallexample
+@smallexample @c output:66E3A2C
 10-Dec-01 Checking balance      Assets:Checking          $ 1,000.00   $ 1,000.00
-                                Equity:Opening Balances $ -1,000.00            0
+                                Equit:Opening Balances  $ -1,000.00            0
 10-Dec-20 Organic Co-op         Expense:Food:Groceries      $ 37.50      $ 37.50
                                 Expense:Food:Groceries      $ 37.50      $ 75.00
                                 Expense:Food:Groceries      $ 37.50     $ 112.50
@@ -450,8 +453,11 @@ interested in seeing transactions for:
 @cindex accounts, limiting by
 @cindex limiting by accounts
 
-@smallexample
+@smallexample @c command:96B0EB3
 $ ledger -f drewr3.dat register Groceries
+@end smallexample
+
+@smallexample @c output:96B0EB3
 10-Dec-20 Organic Co-op         Expense:Food:Groceries      $ 37.50      $ 37.50
                                 Expense:Food:Groceries      $ 37.50      $ 75.00
                                 Expense:Food:Groceries      $ 37.50     $ 112.50
@@ -465,8 +471,11 @@ $ ledger -f drewr3.dat register Groceries
 @noindent
 Which matches the balance reported for the @samp{Groceries} account:
 
-@smallexample
+@smallexample @c command:AECD64E
 $ ledger -f drewr3.dat balance Groceries
+@end smallexample
+
+@smallexample @c output:AECD64E
             $ 334.00  Expenses:Food:Groceries
 @end smallexample
 
@@ -474,8 +483,11 @@ $ ledger -f drewr3.dat balance Groceries
 If you would like to find transaction to only a certain payee use
 @samp{payee} or @samp{@@}:
 
-@smallexample
+@smallexample @c command:C6BC57E
 $ ledger -f drewr3.dat register payee "Organic"
+@end smallexample
+
+@smallexample @c output:C10BC57E
 10-Dec-20 Organic Co-op         Expense:Food:Groceries      $ 37.50      $ 37.50
                                 Expense:Food:Groceries      $ 37.50      $ 75.00
                                 Expense:Food:Groceries      $ 37.50     $ 112.50
@@ -497,8 +509,11 @@ a check to clear, but you should treat it as money spent.  The
 @command{cleared} report will not format correctly for accounts that
 contain multiple commodities):
 
-@smallexample
+@smallexample @c command:B86F6A6
 $ ledger -f drewr3.dat cleared
+@end smallexample
+
+@smallexample @c output:B86F6A6
      $ -3,804.00            $ 775.00                 Assets
       $ 1,396.00            $ 775.00    10-Dec-20      Checking
          $ 30.00                   0                     Business
@@ -517,8 +532,8 @@ $ ledger -f drewr3.dat cleared
         $ -20.00                   0                   MasterCard
         $ 200.00                   0                   Mortgage:Principal
        $ -243.60                   0                   Tithe
-----------------    ----------------    ---------
-       $ -243.60                   0
+----------------  ----------------    ---------
+       $ -243.60                   0             
 @end smallexample
 
 @noindent
@@ -3826,8 +3841,11 @@ note the implicit logical and between @samp{Auto} and
 If you want the entire contents of a branch of your account tree, use
 the highest common name in the branch:
 
-@smallexample
+@smallexample @c command:B0468E1
 $ ledger balance -f drewr3.dat Income
+@end smallexample
+
+@smallexample @c output:B0468E1
          $ -2,030.00  Income
          $ -2,000.00    Salary
             $ -30.00    Sales
@@ -3838,15 +3856,25 @@ $ ledger balance -f drewr3.dat Income
 You can use general regular expressions in nearly anyplace Ledger
 needs a string:
 
-@smallexample
+@smallexample @c command:EAE389F
 $ ledger balance -f drewr3.dat ^Bo
+@end smallexample
+@smallexample @c output:EAE389F
+@end smallexample
+
+This first example looks for any account starting with @samp{Bo}, of
+which there are none.
+
+@smallexample @c command:E2AF6AD
 $ ledger balance -f drewr3.dat Bo
+@end smallexample
+
+@smallexample @c output:E2AF6AD
              $ 20.00  Expenses:Books
 @end smallexample
 
-The first example looks for any account starting with @samp{Bo}, of
-which there are none.  The second looks for any account with @samp{Bo},
-which is @samp{Expenses:Books}.
+This second example looks for any account with @samp{Bo}, which is
+@samp{Expenses:Books}.
 
 @cindex limit by payees
 @findex --limit @var{EXPR}
@@ -5596,22 +5624,23 @@ Format Codes}).
 @item --master-account @var{STR}
 Prepend all account names with the argument.
 
-@smallexample
-$ ledger -f test/input/drewr3.dat bal --master-account HUMBUG
+@smallexample @c command:A76BB56
+$ ledger -f drewr3.dat bal --no-total --master-account HUMBUG
+@end smallexample
+
+@smallexample @c output:A76BB56
                    0  HUMBUG
          $ -3,804.00    Assets
           $ 1,396.00      Checking
              $ 30.00        Business
          $ -5,200.00      Savings
-             $ 20.00    Books
          $ -1,000.00    Equity:Opening Balances
-          $ 6,634.00    Expenses
-         $ 11,000.00      Auto
+          $ 6,654.00    Expenses
+          $ 5,500.00      Auto
              $ 20.00      Books
             $ 300.00      Escrow
             $ 334.00      Food:Groceries
             $ 500.00      Interest:Mortgage
-         $ -5,520.00      Assets:Checking
          $ -2,030.00    Income
          $ -2,000.00      Salary
             $ -30.00      Sales
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 94ce0a0a..159ab5be 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -38,4 +38,16 @@ add_subdirectory(manual)
 add_subdirectory(baseline)
 add_subdirectory(regress)
 
+if(PYTHONINTERP_FOUND)
+  set(_class DocTests)
+  file(GLOB ${_class}_TESTS ${PROJECT_SOURCE_DIR}/doc/*.texi)
+  foreach(TestFile ${${_class}_TESTS})
+    get_filename_component(TestFile_Name ${TestFile} NAME_WE)
+    add_test(${_class}Test_${TestFile_Name}
+      ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/DocTests.py
+      ${LEDGER_LOCATION} ${TestFile})
+    set_target_properties(check PROPERTIES DEPENDS ${_class}Test_${TestFile_Name})
+  endforeach()
+endif()
+
 ### CMakeLists.txt ends here
diff --git a/test/DocTests.py b/test/DocTests.py
new file mode 100755
index 00000000..daac1db5
--- /dev/null
+++ b/test/DocTests.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import sys
+import hashlib
+import subprocess
+
+from difflib import unified_diff
+
+class DocTests:
+  def __init__(self, argv):
+    if not os.path.isfile(argv[1]):
+        print "Cannot find ledger at '%s'" % argv[1]
+        sys.exit(1)
+    if not os.path.isfile(argv[2]):
+        print "Cannot find source path at '%s'" % argv[2]
+        sys.exit(1)
+
+    self.ledger     = os.path.abspath(argv[1])
+    self.sourcepath = os.path.abspath(argv[2])
+    scriptpath = os.path.dirname(os.path.realpath(__file__))
+    self.verbose = False
+    self.debug = False
+    self.examples = dict()
+    self.testin_token = 'command'
+    self.testout_token = 'output'
+
+  def read_example(self):
+    endexample = re.compile(r'^@end\s+smallexample\s*$')
+    example = str()
+    while True:
+      line = self.file.readline()
+      self.current_line += 1
+      if len(line) <= 0 or endexample.match(line): break
+      example += line
+    return example
+
+  def test_id(self, example):
+    return hashlib.sha1(example.rstrip()).hexdigest()[0:7].upper()
+
+  def find_examples(self):
+    startexample = re.compile(r'^@smallexample\s+@c\s+(%s|%s)(?::([\dA-Fa-f]+))?' % (self.testin_token, self.testout_token))
+    while True:
+      line = self.file.readline()
+      self.current_line += 1
+      if len(line) <= 0: break
+
+      startmatch = startexample.match(line)
+      if (startmatch):
+        test_begin_pos = self.file.tell()
+        test_begin_line = self.current_line
+        test_kind = startmatch.group(1)
+        test_id = startmatch.group(2)
+        example = self.read_example()
+        test_end_pos = self.file.tell()
+        test_end_line = self.current_line
+
+        if not test_id:
+          print >> sys.stderr, 'Example', test_kind, 'in line', test_begin_line, 'is missing id.'
+          test_id = self.test_id(example)
+          if test_kind == self.testin_token:
+            print >> sys.stderr, 'Use', self.test_id(example)
+        elif test_kind == self.testin_token and test_id != self.test_id(example):
+          print >> sys.stderr, 'Expected test id', test_id, 'for example' \
+              , test_kind, 'on line', test_begin_line, 'to be', self.test_id(example)
+
+        try:
+          self.examples[test_id]
+        except KeyError:
+          self.examples[test_id] = dict()
+
+        self.examples[test_id][test_kind] = {
+            'bpos': test_begin_pos,
+            'epos': test_end_pos,
+            'blin': test_begin_line,
+            'elin': test_end_line,
+            test_kind: example,
+            }
+
+  def test_examples(self):
+    failed = 0
+    for test_id in self.examples:
+      example = self.examples[test_id]
+      try:
+        command = example[self.testin_token][self.testin_token]
+      except KeyError:
+        command = None
+      try:
+        output = example[self.testout_token][self.testout_token]
+      except KeyError:
+        output = None
+
+      if command and output:
+        command = command.rstrip().split()
+        if command[0] == '$': command.remove('$')
+        index = command.index('ledger')
+        command[index] = self.ledger
+        command.insert(index+1, '--init-file')
+        command.insert(index+2, '/dev/null')
+        scriptpath = os.path.dirname(os.path.realpath(__file__))
+        test_input_dir = scriptpath + '/../test/input/'
+        for i, arg in enumerate(command):
+          if '.dat' in arg or '.ledger' in arg:
+            if os.path.exists(test_input_dir + arg):
+              command[i] = test_input_dir + arg
+        try:
+          verify = subprocess.check_output(command)#.decode('utf-8')
+        except:
+          verify = str()
+        valid = (output == verify)
+        if self.verbose:
+          print test_id, ':', u'Passed' if valid else u'FAILED'
+        else:
+          sys.stdout.write('.' if valid else 'E')
+
+        if not valid:
+          failed += 1
+          if self.debug:
+            print ' '.join(command)
+            for line in unified_diff(output.split('\n'), verify.split('\n'), fromfile='generated', tofile='expected'):
+              print(line)
+            print
+    print
+    return failed
+
+  def main(self):
+    self.file = open(self.sourcepath)
+    self.current_line = 0
+    self.find_examples()
+    failed_examples = self.test_examples()
+    self.file.close()
+    return failed_examples
+
+if __name__ == "__main__":
+  script = DocTests(sys.argv)
+  status = script.main()
+  sys.exit(status)