OILS / test / spec_lib.py View on Github | oilshell.org

226 lines, 164 significant
1"""
2spec_lib.py
3
4Shared between sh_spec.py (Python 2) and spec/stateful/harness.py (Python 3)!
5"""
6from __future__ import print_function
7
8import optparse
9import os
10import re
11import sys
12
13
14def log(msg, *args):
15 # type: (str, *Any) -> None
16 if args:
17 msg = msg % args
18 print(msg, file=sys.stderr)
19
20
21# Note that devtools/release.sh spec-all runs with bin/osh and $DIR/_bin/osh,
22# which should NOT match
23
24OSH_CPP_RE = re.compile(r'_bin/\w+-\w+(-sh)?/osh') # e.g. $PWD/_bin/cxx-dbg/osh
25YSH_CPP_RE = re.compile(r'_bin/\w+-\w+(-sh)?/ysh') # e.g. $PWD/_bin/cxx-dbg/ysh
26OIL_CPP_RE = re.compile(r'_bin/\w+-\w+(-sh)?/oil')
27
28def MakeShellPairs(shells):
29 shell_pairs = []
30
31 saw_osh = False
32 saw_ysh = False
33 saw_oil = False
34
35 for path in shells:
36 name, _ = os.path.splitext(path)
37 label = os.path.basename(name)
38
39 if label == 'osh':
40 # change the second 'osh' to 'osh_ALT' so it's distinct
41 if saw_osh:
42 if OSH_CPP_RE.search(path):
43 label = 'osh-cpp'
44 else:
45 label = 'osh_ALT'
46 saw_osh = True
47
48 elif label == 'ysh':
49 if saw_ysh:
50 if YSH_CPP_RE.search(path):
51 label = 'ysh-cpp'
52 else:
53 label = 'ysh_ALT'
54
55 saw_ysh = True
56
57 elif label == 'oil': # TODO: remove this
58 if saw_oil:
59 if OIL_CPP_RE.search(path):
60 label = 'oil-cpp'
61 else:
62 label = 'oil_ALT'
63
64 saw_oil = True
65
66 shell_pairs.append((label, path))
67 return shell_pairs
68
69
70RANGE_RE = re.compile('(\d+) \s* - \s* (\d+)', re.VERBOSE)
71
72
73def ParseRange(range_str):
74 try:
75 d = int(range_str)
76 return d, d # singleton range
77 except ValueError:
78 m = RANGE_RE.match(range_str)
79 if not m:
80 raise RuntimeError('Invalid range %r' % range_str)
81 b, e = m.groups()
82 return int(b), int(e)
83
84
85class RangePredicate(object):
86 """Zero-based indexing, inclusive ranges."""
87
88 def __init__(self, begin, end):
89 self.begin = begin
90 self.end = end
91
92 def __call__(self, i, case):
93 return self.begin <= i <= self.end
94
95
96class RegexPredicate(object):
97 """Filter by name."""
98
99 def __init__(self, desc_re):
100 self.desc_re = desc_re
101
102 def __call__(self, i, case):
103 return bool(self.desc_re.search(case['desc']))
104
105
106
107def DefineCommon(p):
108 """Flags shared between sh_spec.py and stateful/harness.py."""
109 p.add_option(
110 '-v', '--verbose', dest='verbose', action='store_true', default=False,
111 help='Show details about test failures')
112 p.add_option(
113 '-r', '--range', dest='range', default=None,
114 help='Execute only a given test range, e.g. 5-10, 5-, -10, or 5')
115 p.add_option(
116 '--regex', dest='regex', default=None,
117 help='Execute only tests whose description matches a given regex '
118 '(case-insensitive)')
119 p.add_option(
120 '--list', dest='do_list', action='store_true', default=None,
121 help='Just list tests')
122 p.add_option(
123 '--oils-failures-allowed', dest='oils_failures_allowed', type='int',
124 default=0, help="Allow this number of Oils failures")
125
126 # Select what shells to run
127 p.add_option(
128 '--oils-bin-dir', dest='oils_bin_dir', default=None,
129 help="Directory that osh and ysh live in")
130 p.add_option(
131 '--oils-cpp-bin-dir', dest='oils_cpp_bin_dir', default=None,
132 help="Directory that native C++ osh and ysh live in")
133 p.add_option(
134 '--ovm-bin-dir', dest='ovm_bin_dir', default=None,
135 help="Directory of the legacy OVM/CPython build")
136 p.add_option(
137 '--compare-shells', dest='compare_shells', action='store_true',
138 help="Compare against shells specified at the top of each file")
139
140
141def DefineStateful(p):
142 p.add_option(
143 '--num-retries', dest='num_retries',
144 type='int', default=4,
145 help='Number of retries (for spec/stateful only)')
146 p.add_option(
147 '--pexpect-timeout', dest='pexpect_timeout',
148 type='float', default=1.0,
149 help='In seconds')
150 p.add_option(
151 '--results-file', dest='results_file', default=None,
152 help='Write table of results to this file. Default is stdout.')
153
154
155def DefineShSpec(p):
156 p.add_option(
157 '-d', '--details', dest='details', action='store_true', default=False,
158 help='Show details even for successful cases (requires -v)')
159 p.add_option(
160 '-t', '--trace', dest='trace', action='store_true', default=False,
161 help='trace execution of shells to diagnose hangs')
162
163 # Execution modes
164 p.add_option(
165 '-p', '--print', dest='do_print', action='store_true', default=None,
166 help="Print test code, but don't run it")
167 p.add_option(
168 '--print-spec-suite', dest='print_spec_suite', action='store_true', default=None,
169 help="Print suite this file belongs to")
170 p.add_option(
171 '--print-table', dest='print_table', action='store_true', default=None,
172 help="Print table of test files")
173 p.add_option(
174 '--print-tagged', dest='print_tagged',
175 help="Print spec files tagged with a certain string")
176
177 # Output control
178 p.add_option(
179 '--format', dest='format', choices=['ansi', 'html'],
180 default='ansi', help="Output format (default 'ansi')")
181 p.add_option(
182 '--stats-file', dest='stats_file', default=None,
183 help="File to write stats to")
184 p.add_option(
185 '--tsv-output', dest='tsv_output', default=None,
186 help="Write a TSV log to this file. Subsumes --stats-file.")
187 p.add_option(
188 '--stats-template', dest='stats_template', default='',
189 help="Python format string for stats")
190
191 p.add_option(
192 '--path-env', dest='path_env', default='',
193 help="The full PATH, for finding binaries used in tests.")
194 p.add_option(
195 '--tmp-env', dest='tmp_env', default='',
196 help="A temporary directory that the tests can use.")
197
198 # Notes:
199 # - utf-8 is the Ubuntu default
200 # - this flag has limited usefulness. It may be better to simply export LANG=
201 # in this test case itself.
202 p.add_option(
203 '--lang-env', dest='lang_env', default='en_US.UTF-8',
204 help="The LANG= setting, which affects various libc functions.")
205 p.add_option(
206 '--env-pair', dest='env_pair', default=[], action='append',
207 help='A key=value pair to add to the environment')
208
209 p.add_option(
210 '--timeout', dest='timeout', default='',
211 help="Prefix shell invocation with 'timeout N'")
212 p.add_option(
213 '--timeout-bin', dest='timeout_bin', default=None,
214 help="Use the smoosh timeout binary at this location.")
215
216 p.add_option(
217 '--posix', dest='posix', default=False, action='store_true',
218 help='Pass -o posix to the shell (when applicable)')
219
220 p.add_option(
221 '--sh-env-var-name', dest='sh_env_var_name', default='SH',
222 help="Set this environment variable to the path of the shell")
223
224 p.add_option(
225 '--pyann-out-dir', dest='pyann_out_dir', default=None,
226 help='Run OSH with PYANN_OUT=$dir/$case_num.json')