OILS / test / spec_lib.py View on Github | oilshell.org

233 lines, 167 significant
1"""
2spec_lib.py
3
4Shared between sh_spec.py (Python 2) and spec/stateful/harness.py (Python 3)!
5"""
6from __future__ import print_function
7
8import os
9import re
10import sys
11
12
13def log(msg, *args):
14 # type: (str, *Any) -> None
15 if args:
16 msg = msg % args
17 print(msg, file=sys.stderr)
18
19
20# Note that devtools/release.sh spec-all runs with bin/osh and $DIR/_bin/osh,
21# which should NOT match
22
23OSH_CPP_RE = re.compile(r'_bin/\w+-\w+(-sh)?/osh') # e.g. $PWD/_bin/cxx-dbg/osh
24YSH_CPP_RE = re.compile(r'_bin/\w+-\w+(-sh)?/ysh') # e.g. $PWD/_bin/cxx-dbg/ysh
25OIL_CPP_RE = re.compile(r'_bin/\w+-\w+(-sh)?/oil')
26
27# e.g. bash-4.4 bash 5.2
28BASH_RE = re.compile(r'bash-\d\.\d$')
29
30def MakeShellPairs(shells):
31 shell_pairs = []
32
33 saw_osh = False
34 saw_ysh = False
35 saw_oil = False
36
37 for path in shells:
38 if BASH_RE.match(path):
39 # Just call it 'bash' for the assertions
40 #label = os.path.basename(path)
41 label = 'bash'
42 else:
43 first, _ = os.path.splitext(path)
44 label = os.path.basename(first)
45
46 if label == 'osh':
47 # change the second 'osh' to 'osh_ALT' so it's distinct
48 if saw_osh:
49 if OSH_CPP_RE.search(path):
50 label = 'osh-cpp'
51 else:
52 label = 'osh_ALT'
53 saw_osh = True
54
55 elif label == 'ysh':
56 if saw_ysh:
57 if YSH_CPP_RE.search(path):
58 label = 'ysh-cpp'
59 else:
60 label = 'ysh_ALT'
61
62 saw_ysh = True
63
64 elif label == 'oil': # TODO: remove this
65 if saw_oil:
66 if OIL_CPP_RE.search(path):
67 label = 'oil-cpp'
68 else:
69 label = 'oil_ALT'
70
71 saw_oil = True
72
73 shell_pairs.append((label, path))
74 return shell_pairs
75
76
77RANGE_RE = re.compile('(\d+) \s* - \s* (\d+)', re.VERBOSE)
78
79
80def ParseRange(range_str):
81 try:
82 d = int(range_str)
83 return d, d # singleton range
84 except ValueError:
85 m = RANGE_RE.match(range_str)
86 if not m:
87 raise RuntimeError('Invalid range %r' % range_str)
88 b, e = m.groups()
89 return int(b), int(e)
90
91
92class RangePredicate(object):
93 """Zero-based indexing, inclusive ranges."""
94
95 def __init__(self, begin, end):
96 self.begin = begin
97 self.end = end
98
99 def __call__(self, i, case):
100 return self.begin <= i <= self.end
101
102
103class RegexPredicate(object):
104 """Filter by name."""
105
106 def __init__(self, desc_re):
107 self.desc_re = desc_re
108
109 def __call__(self, i, case):
110 return bool(self.desc_re.search(case['desc']))
111
112
113
114def DefineCommon(p):
115 """Flags shared between sh_spec.py and stateful/harness.py."""
116 p.add_option(
117 '-v', '--verbose', dest='verbose', action='store_true', default=False,
118 help='Show details about test failures')
119 p.add_option(
120 '-r', '--range', dest='range', default=None,
121 help='Execute only a given test range, e.g. 5-10, 5-, -10, or 5')
122 p.add_option(
123 '--regex', dest='regex', default=None,
124 help='Execute only tests whose description matches a given regex '
125 '(case-insensitive)')
126 p.add_option(
127 '--list', dest='do_list', action='store_true', default=None,
128 help='Just list tests')
129 p.add_option(
130 '--oils-failures-allowed', dest='oils_failures_allowed', type='int',
131 default=0, help="Allow this number of Oils failures")
132
133 # Select what shells to run
134 p.add_option(
135 '--oils-bin-dir', dest='oils_bin_dir', default=None,
136 help="Directory that osh and ysh live in")
137 p.add_option(
138 '--oils-cpp-bin-dir', dest='oils_cpp_bin_dir', default=None,
139 help="Directory that native C++ osh and ysh live in")
140 p.add_option(
141 '--ovm-bin-dir', dest='ovm_bin_dir', default=None,
142 help="Directory of the legacy OVM/CPython build")
143 p.add_option(
144 '--compare-shells', dest='compare_shells', action='store_true',
145 help="Compare against shells specified at the top of each file")
146
147
148def DefineStateful(p):
149 p.add_option(
150 '--num-retries', dest='num_retries',
151 type='int', default=4,
152 help='Number of retries (for spec/stateful only)')
153 p.add_option(
154 '--pexpect-timeout', dest='pexpect_timeout',
155 type='float', default=1.0,
156 help='In seconds')
157 p.add_option(
158 '--results-file', dest='results_file', default=None,
159 help='Write table of results to this file. Default is stdout.')
160
161
162def DefineShSpec(p):
163 p.add_option(
164 '-d', '--details', dest='details', action='store_true', default=False,
165 help='Show details even for successful cases (requires -v)')
166 p.add_option(
167 '-t', '--trace', dest='trace', action='store_true', default=False,
168 help='trace execution of shells to diagnose hangs')
169
170 # Execution modes
171 p.add_option(
172 '-p', '--print', dest='do_print', action='store_true', default=None,
173 help="Print test code, but don't run it")
174 p.add_option(
175 '--print-spec-suite', dest='print_spec_suite', action='store_true', default=None,
176 help="Print suite this file belongs to")
177 p.add_option(
178 '--print-table', dest='print_table', action='store_true', default=None,
179 help="Print table of test files")
180 p.add_option(
181 '--print-tagged', dest='print_tagged',
182 help="Print spec files tagged with a certain string")
183
184 # Output control
185 p.add_option(
186 '--format', dest='format', choices=['ansi', 'html'],
187 default='ansi', help="Output format (default 'ansi')")
188 p.add_option(
189 '--stats-file', dest='stats_file', default=None,
190 help="File to write stats to")
191 p.add_option(
192 '--tsv-output', dest='tsv_output', default=None,
193 help="Write a TSV log to this file. Subsumes --stats-file.")
194 p.add_option(
195 '--stats-template', dest='stats_template', default='',
196 help="Python format string for stats")
197
198 p.add_option(
199 '--path-env', dest='path_env', default='',
200 help="The full PATH, for finding binaries used in tests.")
201 p.add_option(
202 '--tmp-env', dest='tmp_env', default='',
203 help="A temporary directory that the tests can use.")
204
205 # Notes:
206 # - utf-8 is the Ubuntu default
207 # - this flag has limited usefulness. It may be better to simply export LANG=
208 # in this test case itself.
209 p.add_option(
210 '--lang-env', dest='lang_env', default='en_US.UTF-8',
211 help="The LANG= setting, which affects various libc functions.")
212 p.add_option(
213 '--env-pair', dest='env_pair', default=[], action='append',
214 help='A key=value pair to add to the environment')
215
216 p.add_option(
217 '--timeout', dest='timeout', default='',
218 help="Prefix shell invocation with 'timeout N'")
219 p.add_option(
220 '--timeout-bin', dest='timeout_bin', default=None,
221 help="Use the smoosh timeout binary at this location.")
222
223 p.add_option(
224 '--posix', dest='posix', default=False, action='store_true',
225 help='Pass -o posix to the shell (when applicable)')
226
227 p.add_option(
228 '--sh-env-var-name', dest='sh_env_var_name', default='SH',
229 help="Set this environment variable to the path of the shell")
230
231 p.add_option(
232 '--pyann-out-dir', dest='pyann_out_dir', default=None,
233 help='Run OSH with PYANN_OUT=$dir/$case_num.json')