OILS / test / spec_lib.py View on Github | oilshell.org

286 lines, 222 significant
1"""spec_lib.py.
2
3Shared between sh_spec.py (Python 2) and spec/stateful/harness.py
4(Python 3)!
5"""
6from __future__ import print_function
7
8import os
9import re
10import sys
11
12
13def log(msg, *args):
14 # type: (str, *Any) -> None
15 if args:
16 msg = msg % args
17 print(msg, file=sys.stderr)
18
19
20# Note that devtools/release.sh spec-all runs with bin/osh and $DIR/_bin/osh,
21# which should NOT match
22
23OSH_CPP_RE = re.compile(
24 r'_bin/\w+-\w+(-sh)?/osh') # e.g. $PWD/_bin/cxx-dbg/osh
25YSH_CPP_RE = re.compile(
26 r'_bin/\w+-\w+(-sh)?/ysh') # e.g. $PWD/_bin/cxx-dbg/ysh
27OIL_CPP_RE = re.compile(r'_bin/\w+-\w+(-sh)?/oil')
28
29# e.g. bash-4.4 bash 5.2.21
30BASH_RE = re.compile(r'(bash-\d)[\d.]+$')
31
32
33def MakeShellPairs(shells):
34 shell_pairs = []
35
36 saw_osh = False
37 saw_ysh = False
38 saw_oil = False
39
40 for path in shells:
41 m = BASH_RE.match(path)
42 if m:
43 label = m.group(1) # bash-4 or to fit
44 else:
45 first, _ = os.path.splitext(path)
46 label = os.path.basename(first)
47
48 if label == 'osh':
49 # change the second 'osh' to 'osh_ALT' so it's distinct
50 if saw_osh:
51 if OSH_CPP_RE.search(path):
52 label = 'osh-cpp'
53 else:
54 label = 'osh_ALT'
55 saw_osh = True
56
57 elif label == 'ysh':
58 if saw_ysh:
59 if YSH_CPP_RE.search(path):
60 label = 'ysh-cpp'
61 else:
62 label = 'ysh_ALT'
63
64 saw_ysh = True
65
66 elif label == 'oil': # TODO: remove this
67 if saw_oil:
68 if OIL_CPP_RE.search(path):
69 label = 'oil-cpp'
70 else:
71 label = 'oil_ALT'
72
73 saw_oil = True
74
75 shell_pairs.append((label, path))
76 return shell_pairs
77
78
79RANGE_RE = re.compile('(\d+) \s* - \s* (\d+)', re.VERBOSE)
80
81
82def ParseRange(range_str):
83 try:
84 d = int(range_str)
85 return d, d # singleton range
86 except ValueError:
87 m = RANGE_RE.match(range_str)
88 if not m:
89 raise RuntimeError('Invalid range %r' % range_str)
90 b, e = m.groups()
91 return int(b), int(e)
92
93
94class RangePredicate(object):
95 """Zero-based indexing, inclusive ranges."""
96
97 def __init__(self, begin, end):
98 self.begin = begin
99 self.end = end
100
101 def __call__(self, i, case):
102 return self.begin <= i <= self.end
103
104
105class RegexPredicate(object):
106 """Filter by name."""
107
108 def __init__(self, desc_re):
109 self.desc_re = desc_re
110
111 def __call__(self, i, case):
112 return bool(self.desc_re.search(case['desc']))
113
114
115def DefineCommon(p):
116 """Flags shared between sh_spec.py and stateful/harness.py."""
117 p.add_option('-v',
118 '--verbose',
119 dest='verbose',
120 action='store_true',
121 default=False,
122 help='Show details about test failures')
123 p.add_option(
124 '-r',
125 '--range',
126 dest='range',
127 default=None,
128 help='Execute only a given test range, e.g. 5-10, 5-, -10, or 5')
129 p.add_option(
130 '--regex',
131 dest='regex',
132 default=None,
133 help='Execute only tests whose description matches a given regex '
134 '(case-insensitive)')
135 p.add_option('--list',
136 dest='do_list',
137 action='store_true',
138 default=None,
139 help='Just list tests')
140 p.add_option('--oils-failures-allowed',
141 dest='oils_failures_allowed',
142 type='int',
143 default=0,
144 help="Allow this number of Oils failures")
145
146 # Select what shells to run
147 p.add_option('--oils-bin-dir',
148 dest='oils_bin_dir',
149 default=None,
150 help="Directory that osh and ysh live in")
151 p.add_option('--oils-cpp-bin-dir',
152 dest='oils_cpp_bin_dir',
153 default=None,
154 help="Directory that native C++ osh and ysh live in")
155 p.add_option('--ovm-bin-dir',
156 dest='ovm_bin_dir',
157 default=None,
158 help="Directory of the legacy OVM/CPython build")
159 p.add_option(
160 '--compare-shells',
161 dest='compare_shells',
162 action='store_true',
163 help="Compare against shells specified at the top of each file")
164
165
166def DefineStateful(p):
167 p.add_option('--num-retries',
168 dest='num_retries',
169 type='int',
170 default=4,
171 help='Number of retries (for spec/stateful only)')
172 p.add_option('--pexpect-timeout',
173 dest='pexpect_timeout',
174 type='float',
175 default=1.0,
176 help='In seconds')
177 p.add_option(
178 '--results-file',
179 dest='results_file',
180 default=None,
181 help='Write table of results to this file. Default is stdout.')
182
183
184def DefineShSpec(p):
185 p.add_option('-d',
186 '--details',
187 dest='details',
188 action='store_true',
189 default=False,
190 help='Show details even for successful cases (requires -v)')
191 p.add_option('-t',
192 '--trace',
193 dest='trace',
194 action='store_true',
195 default=False,
196 help='trace execution of shells to diagnose hangs')
197
198 # Execution modes
199 p.add_option('-p',
200 '--print',
201 dest='do_print',
202 action='store_true',
203 default=None,
204 help="Print test code, but don't run it")
205 p.add_option('--print-spec-suite',
206 dest='print_spec_suite',
207 action='store_true',
208 default=None,
209 help="Print suite this file belongs to")
210 p.add_option('--print-table',
211 dest='print_table',
212 action='store_true',
213 default=None,
214 help="Print table of test files")
215 p.add_option('--print-tagged',
216 dest='print_tagged',
217 help="Print spec files tagged with a certain string")
218
219 # Output control
220 p.add_option('--format',
221 dest='format',
222 choices=['ansi', 'html'],
223 default='ansi',
224 help="Output format (default 'ansi')")
225 p.add_option('--stats-file',
226 dest='stats_file',
227 default=None,
228 help="File to write stats to")
229 p.add_option('--tsv-output',
230 dest='tsv_output',
231 default=None,
232 help="Write a TSV log to this file. Subsumes --stats-file.")
233 p.add_option('--stats-template',
234 dest='stats_template',
235 default='',
236 help="Python format string for stats")
237
238 p.add_option('--path-env',
239 dest='path_env',
240 default='',
241 help="The full PATH, for finding binaries used in tests.")
242 p.add_option('--tmp-env',
243 dest='tmp_env',
244 default='',
245 help="A temporary directory that the tests can use.")
246
247 # Notes:
248 # - utf-8 is the Ubuntu default
249 # - this flag has limited usefulness. It may be better to simply export LANG=
250 # in this test case itself.
251 if 0:
252 p.add_option(
253 '--lang-env',
254 dest='lang_env',
255 default='en_US.UTF-8',
256 help="The LANG= setting, which affects various libc functions.")
257 p.add_option('--env-pair',
258 dest='env_pair',
259 default=[],
260 action='append',
261 help='A key=value pair to add to the environment')
262
263 p.add_option('--timeout',
264 dest='timeout',
265 default='',
266 help="Prefix shell invocation with 'timeout N'")
267 p.add_option('--timeout-bin',
268 dest='timeout_bin',
269 default=None,
270 help="Use the smoosh timeout binary at this location.")
271
272 p.add_option('--posix',
273 dest='posix',
274 default=False,
275 action='store_true',
276 help='Pass -o posix to the shell (when applicable)')
277
278 p.add_option('--sh-env-var-name',
279 dest='sh_env_var_name',
280 default='SH',
281 help="Set this environment variable to the path of the shell")
282
283 p.add_option('--pyann-out-dir',
284 dest='pyann_out_dir',
285 default=None,
286 help='Run OSH with PYANN_OUT=$dir/$case_num.json')