From fbbb379fe08b051b40c071041108a2526533f417 Mon Sep 17 00:00:00 2001 From: Alexis Hildebrandt Date: Fri, 7 Feb 2014 00:21:38 +0100 Subject: Check examples in documentation when running tests The DocTests.py script will parse a given texinfo file for specially marked examples, run the ledger command from the example, and check the result against the example output from the documentation. --- test/DocTests.py | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100755 test/DocTests.py (limited to 'test/DocTests.py') diff --git a/test/DocTests.py b/test/DocTests.py new file mode 100755 index 00000000..daac1db5 --- /dev/null +++ b/test/DocTests.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import sys +import hashlib +import subprocess + +from difflib import unified_diff + +class DocTests: + def __init__(self, argv): + if not os.path.isfile(argv[1]): + print "Cannot find ledger at '%s'" % argv[1] + sys.exit(1) + if not os.path.isfile(argv[2]): + print "Cannot find source path at '%s'" % argv[2] + sys.exit(1) + + self.ledger = os.path.abspath(argv[1]) + self.sourcepath = os.path.abspath(argv[2]) + scriptpath = os.path.dirname(os.path.realpath(__file__)) + self.verbose = False + self.debug = False + self.examples = dict() + self.testin_token = 'command' + self.testout_token = 'output' + + def read_example(self): + endexample = re.compile(r'^@end\s+smallexample\s*$') + example = str() + while True: + line = self.file.readline() + self.current_line += 1 + if len(line) <= 0 or endexample.match(line): break + example += line + return example + + def test_id(self, example): + return hashlib.sha1(example.rstrip()).hexdigest()[0:7].upper() + + def find_examples(self): + startexample = re.compile(r'^@smallexample\s+@c\s+(%s|%s)(?::([\dA-Fa-f]+))?' % (self.testin_token, self.testout_token)) + while True: + line = self.file.readline() + self.current_line += 1 + if len(line) <= 0: break + + startmatch = startexample.match(line) + if (startmatch): + test_begin_pos = self.file.tell() + test_begin_line = self.current_line + test_kind = startmatch.group(1) + test_id = startmatch.group(2) + example = self.read_example() + test_end_pos = self.file.tell() + test_end_line = self.current_line + + if not test_id: + print >> sys.stderr, 'Example', test_kind, 'in line', test_begin_line, 'is missing id.' + test_id = self.test_id(example) + if test_kind == self.testin_token: + print >> sys.stderr, 'Use', self.test_id(example) + elif test_kind == self.testin_token and test_id != self.test_id(example): + print >> sys.stderr, 'Expected test id', test_id, 'for example' \ + , test_kind, 'on line', test_begin_line, 'to be', self.test_id(example) + + try: + self.examples[test_id] + except KeyError: + self.examples[test_id] = dict() + + self.examples[test_id][test_kind] = { + 'bpos': test_begin_pos, + 'epos': test_end_pos, + 'blin': test_begin_line, + 'elin': test_end_line, + test_kind: example, + } + + def test_examples(self): + failed = 0 + for test_id in self.examples: + example = self.examples[test_id] + try: + command = example[self.testin_token][self.testin_token] + except KeyError: + command = None + try: + output = example[self.testout_token][self.testout_token] + except KeyError: + output = None + + if command and output: + command = command.rstrip().split() + if command[0] == '$': command.remove('$') + index = command.index('ledger') + command[index] = self.ledger + command.insert(index+1, '--init-file') + command.insert(index+2, '/dev/null') + scriptpath = os.path.dirname(os.path.realpath(__file__)) + test_input_dir = scriptpath + '/../test/input/' + for i, arg in enumerate(command): + if '.dat' in arg or '.ledger' in arg: + if os.path.exists(test_input_dir + arg): + command[i] = test_input_dir + arg + try: + verify = subprocess.check_output(command)#.decode('utf-8') + except: + verify = str() + valid = (output == verify) + if self.verbose: + print test_id, ':', u'Passed' if valid else u'FAILED' + else: + sys.stdout.write('.' if valid else 'E') + + if not valid: + failed += 1 + if self.debug: + print ' '.join(command) + for line in unified_diff(output.split('\n'), verify.split('\n'), fromfile='generated', tofile='expected'): + print(line) + print + print + return failed + + def main(self): + self.file = open(self.sourcepath) + self.current_line = 0 + self.find_examples() + failed_examples = self.test_examples() + self.file.close() + return failed_examples + +if __name__ == "__main__": + script = DocTests(sys.argv) + status = script.main() + sys.exit(status) -- cgit v1.2.3 From a1cc8ca15ae1761fbcb4c6ec9f8af82e8ab411ab Mon Sep 17 00:00:00 2001 From: Alexis Hildebrandt Date: Fri, 7 Feb 2014 18:38:43 +0100 Subject: Add support to check documentation examples with inline data --- doc/ledger3.texi | 58 ++++++++++++++++++++++++++++++++------------------------ test/DocTests.py | 45 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 34 deletions(-) (limited to 'test/DocTests.py') diff --git a/doc/ledger3.texi b/doc/ledger3.texi index acf58698..48fafc13 100644 --- a/doc/ledger3.texi +++ b/doc/ledger3.texi @@ -1503,7 +1503,7 @@ entry. For example, the following entries reflect transaction made for a business trip to Europe from the US: -@smallexample +@smallexample @c input:82150D9 2011/09/23 Cash in Munich Assets:Cash E50.00 Assets:Checking $-66.00 @@ -1519,8 +1519,11 @@ spent on Dinner in Munich. Running a ledger balance report shows: -@smallexample +@smallexample @c command:82150D9 $ ledger -f example.dat bal +@end smallexample + +@smallexample @c output:82150D9 $-66.00 E15.00 Assets E15.00 Cash @@ -3641,7 +3644,7 @@ the money to be evenly distributed over the next six months so that your monthly budgets gradually take a hit for the vegetables you'll pick up from the co-op, even though you've already paid for them. -@smallexample +@smallexample @c input:6453542 2008/10/16 * (2090) Bountiful Blessings Farm Expenses:Food:Groceries $ 37.50 ; [=2008/10/01] Expenses:Food:Groceries $ 37.50 ; [=2008/11/01] @@ -3659,15 +3662,17 @@ really knows that it debited $225 this month. And using @option{--effective} option, initial date will be overridden by effective dates. -@smallexample +@smallexample @c command:6453542 $ ledger --effective register Groceries +@end smallexample -08-Oct-01 Bountiful Blessi.. Expe:Food:Groceries $ 37.50 $ 37.50 -08-Nov-01 Bountiful Blessi.. Expe:Food:Groceries $ 37.50 $ 75.00 -08-Dec-01 Bountiful Blessi.. Expe:Food:Groceries $ 37.50 $ 112.50 -09-Jan-01 Bountiful Blessi.. Expe:Food:Groceries $ 37.50 $ 150.00 -09-Feb-01 Bountiful Blessi.. Expe:Food:Groceries $ 37.50 $ 187.50 -09-Mar-01 Bountiful Blessi.. Expe:Food:Groceries $ 37.50 $ 225.00 +@smallexample @c output:6453542 +08-Oct-01 Bountiful Blessings.. Expense:Food:Groceries $ 37.50 $ 37.50 +08-Nov-01 Bountiful Blessings.. Expense:Food:Groceries $ 37.50 $ 75.00 +08-Dec-01 Bountiful Blessings.. Expense:Food:Groceries $ 37.50 $ 112.50 +09-Jan-01 Bountiful Blessings.. Expense:Food:Groceries $ 37.50 $ 150.00 +09-Feb-01 Bountiful Blessings.. Expense:Food:Groceries $ 37.50 $ 187.50 +09-Mar-01 Bountiful Blessings.. Expense:Food:Groceries $ 37.50 $ 225.00 @end smallexample @node Periodic Transactions, Concrete Example of Automated Transactions, Effective Dates, Automated Transactions @@ -3792,14 +3797,14 @@ options. The balance report is the most commonly used report. The simplest invocation is: -@smallexample +@smallexample @c command:1D00D56 $ ledger balance -f drewr3.dat @end smallexample @noindent which will print the balances of every account in your journal. -@smallexample +@smallexample @c output:1D00D56 $ -3,804.00 Assets $ 1,396.00 Checking $ 30.00 Business @@ -3826,8 +3831,11 @@ Most times this is more than you want. Limiting the results to specific accounts is as easy as entering the names of the accounts after the command. -@smallexample +@smallexample @c command:06B2AD4 $ ledger balance -f drewr3.dat Auto MasterCard +@end smallexample + +@smallexample @c output:06B2AD4 $ 5,500.00 Expenses:Auto $ -20.00 Liabilities:MasterCard -------------------- @@ -4992,7 +5000,7 @@ earlier postings. Here's how it works: Say you currently have this posting in your ledger file: -@smallexample +@smallexample @c input:03ACB97 2004/03/15 * Viva Italiano Expenses:Food $12.45 Expenses:Tips $2.55 @@ -5003,17 +5011,17 @@ Now it's @samp{2004/4/9}, and you've just eating at @samp{Viva Italiano} again. The exact amounts are different, but the overall form is the same. With the @command{xact} command you can type: -@smallexample +@smallexample @c command:03ACB97 $ ledger xact 2004/4/9 viva food 11 tips 2.50 @end smallexample This produces the following output: -@smallexample +@smallexample @c output:03ACB97 2004/04/09 Viva Italiano - Expenses:Food $11.00 - Expenses:Tips $2.50 - Liabilities:MasterCard $-13.50 + Expenses:Food $11.00 + Expenses:Tips $2.50 + Liabilities:MasterCard @end smallexample It works by finding a past posting matching the regular expression @@ -6491,7 +6499,7 @@ In the balance report, it shows all the accounts affected by transactions having a related posting. For example, if a file had this transaction: -@smallexample +@smallexample @c input:94C5675 2004/03/20 Safeway Expenses:Food $65.00 Expenses:Cash $20.00 @@ -6500,16 +6508,16 @@ this transaction: And the register command was: -@smallexample -$ ledger -r register food +@smallexample @c command:94C5675 +$ ledger -f example.dat -r register food @end smallexample The following would be output, showing the postings related to the posting that matched: -@smallexample -2004/03/20 Safeway Expenses:Cash $-20.00 $-20.00 - Assets:Checking $85.00 $65.00 +@smallexample @c output:94C5675 +04-Mar-20 Safeway Expenses:Cash $20.00 $20.00 + Assets:Checking $-85.00 $-65.00 @end smallexample @item --budget diff --git a/test/DocTests.py b/test/DocTests.py index daac1db5..eb1a0205 100755 --- a/test/DocTests.py +++ b/test/DocTests.py @@ -26,6 +26,8 @@ class DocTests: self.examples = dict() self.testin_token = 'command' self.testout_token = 'output' + self.testdat_token = 'input' + self.test_files = list() def read_example(self): endexample = re.compile(r'^@end\s+smallexample\s*$') @@ -41,7 +43,8 @@ class DocTests: return hashlib.sha1(example.rstrip()).hexdigest()[0:7].upper() def find_examples(self): - startexample = re.compile(r'^@smallexample\s+@c\s+(%s|%s)(?::([\dA-Fa-f]+))?' % (self.testin_token, self.testout_token)) + startexample = re.compile(r'^@smallexample\s+@c\s+(%s|%s|%s)(?::([\dA-Fa-f]+))?' + % (self.testin_token, self.testout_token, self.testdat_token)) while True: line = self.file.readline() self.current_line += 1 @@ -87,11 +90,17 @@ class DocTests: command = example[self.testin_token][self.testin_token] except KeyError: command = None + try: output = example[self.testout_token][self.testout_token] except KeyError: output = None + try: + input = example[self.testdat_token][self.testdat_token] + except KeyError: + input = None + if command and output: command = command.rstrip().split() if command[0] == '$': command.remove('$') @@ -99,19 +108,37 @@ class DocTests: command[index] = self.ledger command.insert(index+1, '--init-file') command.insert(index+2, '/dev/null') - scriptpath = os.path.dirname(os.path.realpath(__file__)) - test_input_dir = scriptpath + '/../test/input/' - for i, arg in enumerate(command): - if '.dat' in arg or '.ledger' in arg: - if os.path.exists(test_input_dir + arg): - command[i] = test_input_dir + arg try: - verify = subprocess.check_output(command)#.decode('utf-8') + findex = command.index('-f') + except ValueError: + try: + findex = command.index('--file') + except ValueError: + findex = index+1 + command.insert(findex, '--file') + command.insert(findex+1, test_id + '.dat') + + if findex: + scriptpath = os.path.dirname(os.path.realpath(__file__)) + test_input_dir = scriptpath + '/../test/input/' + test_file = command[findex+1] + test_file_created = False + if not os.path.exists(test_file): + if input: + test_file_created = True + with open(test_file, 'w') as f: + f.write(input) + elif os.path.exists(test_input_dir + test_file): + command[findex+1] = test_input_dir + test_file + try: + verify = subprocess.check_output(command) except: verify = str() + if test_file_created: + os.remove(test_file) valid = (output == verify) if self.verbose: - print test_id, ':', u'Passed' if valid else u'FAILED' + print test_id, ':', 'Passed' if valid else 'FAILED' else: sys.stdout.write('.' if valid else 'E') -- cgit v1.2.3 From 960ebc2a572e3ff90d2b95c54f618430df5db351 Mon Sep 17 00:00:00 2001 From: Alexis Hildebrandt Date: Sun, 9 Feb 2014 07:20:03 +0100 Subject: Print summary list of failed doc tests if any --- test/DocTests.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'test/DocTests.py') diff --git a/test/DocTests.py b/test/DocTests.py index eb1a0205..736be6c7 100755 --- a/test/DocTests.py +++ b/test/DocTests.py @@ -83,7 +83,7 @@ class DocTests: } def test_examples(self): - failed = 0 + failed = set() for test_id in self.examples: example = self.examples[test_id] try: @@ -143,14 +143,17 @@ class DocTests: sys.stdout.write('.' if valid else 'E') if not valid: - failed += 1 if self.debug: + failed.add(test_id) print ' '.join(command) for line in unified_diff(output.split('\n'), verify.split('\n'), fromfile='generated', tofile='expected'): print(line) print print - return failed + if len(failed) > 0: + print "\nThe following examples failed:" + print " ", "\n ".join(failed) + return len(failed) def main(self): self.file = open(self.sourcepath) -- cgit v1.2.3 From c566afe3b1b24d3efea0e14c17a45d0987f42f16 Mon Sep 17 00:00:00 2001 From: Alexis Hildebrandt Date: Sun, 9 Feb 2014 07:28:58 +0100 Subject: Add proper argument parsing to DocTests.py --- test/CMakeLists.txt | 2 +- test/DocTests.py | 56 ++++++++++++++++++++++++++++++++++------------------- 2 files changed, 37 insertions(+), 21 deletions(-) (limited to 'test/DocTests.py') diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 159ab5be..796ef0a2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -45,7 +45,7 @@ if(PYTHONINTERP_FOUND) get_filename_component(TestFile_Name ${TestFile} NAME_WE) add_test(${_class}Test_${TestFile_Name} ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/DocTests.py - ${LEDGER_LOCATION} ${TestFile}) + --ledger ${LEDGER_LOCATION} --file ${TestFile}) set_target_properties(check PROPERTIES DEPENDS ${_class}Test_${TestFile_Name}) endforeach() endif() diff --git a/test/DocTests.py b/test/DocTests.py index 736be6c7..a50ec03d 100755 --- a/test/DocTests.py +++ b/test/DocTests.py @@ -5,29 +5,23 @@ import os import re import sys import hashlib +import argparse import subprocess from difflib import unified_diff class DocTests: - def __init__(self, argv): - if not os.path.isfile(argv[1]): - print "Cannot find ledger at '%s'" % argv[1] - sys.exit(1) - if not os.path.isfile(argv[2]): - print "Cannot find source path at '%s'" % argv[2] - sys.exit(1) - - self.ledger = os.path.abspath(argv[1]) - self.sourcepath = os.path.abspath(argv[2]) - scriptpath = os.path.dirname(os.path.realpath(__file__)) - self.verbose = False - self.debug = False - self.examples = dict() - self.testin_token = 'command' + def __init__(self, args): + scriptpath = os.path.dirname(os.path.realpath(__file__)) + self.ledger = os.path.abspath(args.ledger) + self.sourcepath = os.path.abspath(args.file) + self.verbose = args.verbose + + self.examples = dict() + self.test_files = list() + self.testin_token = 'command' self.testout_token = 'output' self.testdat_token = 'input' - self.test_files = list() def read_example(self): endexample = re.compile(r'^@end\s+smallexample\s*$') @@ -137,19 +131,20 @@ class DocTests: if test_file_created: os.remove(test_file) valid = (output == verify) - if self.verbose: + if self.verbose > 0: print test_id, ':', 'Passed' if valid else 'FAILED' else: sys.stdout.write('.' if valid else 'E') if not valid: - if self.debug: failed.add(test_id) + if self.verbose > 1: print ' '.join(command) for line in unified_diff(output.split('\n'), verify.split('\n'), fromfile='generated', tofile='expected'): print(line) print - print + if not self.verbose: + print if len(failed) > 0: print "\nThe following examples failed:" print " ", "\n ".join(failed) @@ -164,6 +159,27 @@ class DocTests: return failed_examples if __name__ == "__main__": - script = DocTests(sys.argv) + def getargs(): + parser = argparse.ArgumentParser(description='DocTests', prefix_chars='-') + parser.add_argument('-v', '--verbose', + dest='verbose', + action='count', + help='be verbose. Add -vv for more verbosity') + parser.add_argument('-l', '--ledger', + dest='ledger', + type=str, + action='store', + required=True, + help='the path to the ledger executable to test with') + parser.add_argument('-f', '--file', + dest='file', + type=str, + action='store', + required=True, + help='the texinfo documentation file to run the examples from') + return parser.parse_args() + + args = getargs() + script = DocTests(args) status = script.main() sys.exit(status) -- cgit v1.2.3