summaryrefslogtreecommitdiff
path: root/test/RegressTests.py
blob: 48efaaa1287575b081846295607ca7e4b7e3e722 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
#!/usr/bin/env python

import sys
import os
import re
import tempfile

from string import join
from difflib import unified_diff

from LedgerHarness import LedgerHarness

harness = LedgerHarness(sys.argv)
tests   = sys.argv[2]

if not os.path.isdir(tests) and not os.path.isfile(tests):
    sys.exit(1)

def test_regression(test_file):
    bug     = open(test_file)
    command = bug.readline()

    line = bug.readline()
    assert "<<<\n" == line
    line = bug.readline()

    data = []
    while line != ">>>1\n":
        data.append(line)
        line = bug.readline()
    line = bug.readline()

    use_stdin = False
    if command.startswith("-f - "):
        use_stdin = True

        command = '$ledger ' + command
    else:
        tempdata = tempfile.mkstemp()

        os.write(tempdata[0], join(data, ''))
        os.close(tempdata[0])

        command = ('$ledger -f "%s" ' % tempdata[1]) + command

    output = []
    while line != ">>>2\n":
        output.append(line)
        line = bug.readline()
    line = bug.readline()

    error = []
    while not line.startswith("==="):
        error.append(line)
        line = bug.readline()

    match = re.match('=== ([0-9]+)', line)
    assert match

    exitcode = int(match.group(1))

    p = harness.run(command, columns=(not re.search('--columns', command)))

    if use_stdin:
        p.stdin.write(join(data))
    p.stdin.close()

    success = True
    printed = False
    index   = 0
    for line in unified_diff(output, harness.readlines(p.stdout)):
        index += 1
        if index < 3:
            continue
        if not printed:
            if success: print
            print "Regression failure in output from %s:" % \
                os.path.basename(test_file)
            success = False
            printed = True
        print " ", line,

    printed = False
    index   = 0
    for line in unified_diff([re.sub('\$FILE', tempdata[1], line)
                              for line in error], harness.readlines(p.stderr)):
        index += 1
        if index < 3:
            continue
        if not printed:
            if success: print
            print "Regression failure in error output from %s:" % \
                os.path.basename(test_file)
            success = False
            printed = True
        print " ", line,

    if exitcode == p.wait():
        if success:
            harness.success()
        else:
            harness.failure()
    else:
        if success: print
        print "Regression failure in exitcode from %s: %d (expected) != %d" % \
            (os.path.basename(test_file), exitcode, p.returncode)
        harness.failure()

    if not use_stdin:
        os.remove(tempdata[1])

if os.path.isdir(tests):
    for test in os.listdir(tests):
        if re.search('\.test$', test):
            test_regression(os.path.join(tests, test))
else:
    test_regression(tests)

harness.exit()