OILS / test / spec_lib.py View on Github | oils.pub

301 lines, 234 significant
1"""spec_lib.py.
2
3Shared between sh_spec.py (Python 2) and spec/stateful/harness.py
4(Python 3)!
5"""
6from __future__ import print_function
7
8import os
9import re
10import sys
11
12
13def log(msg, *args):
14 # type: (str, *Any) -> None
15 if args:
16 msg = msg % args
17 print(msg, file=sys.stderr)
18
19
20# Note that devtools/release.sh spec-all runs with bin/osh and $DIR/_bin/osh,
21# which should NOT match
22
23OSH_CPP_RE = re.compile(
24 r'_bin/\w+-\w+(-sh)?/osh') # e.g. $PWD/_bin/cxx-dbg/osh
25YSH_CPP_RE = re.compile(
26 r'_bin/\w+-\w+(-sh)?/ysh') # e.g. $PWD/_bin/cxx-dbg/ysh
27OIL_CPP_RE = re.compile(r'_bin/\w+-\w+(-sh)?/oil')
28
29# e.g. bash-4.4 bash 5.2.21
30BASH_RE = re.compile(r'(bash-\d)[\d.]+$')
31
32
33def MakeShellPairs(shells):
34 shell_pairs = []
35
36 saw_osh = False
37 saw_ysh = False
38 saw_oil = False
39
40 for path in shells:
41 m = BASH_RE.match(path)
42 if m:
43 label = m.group(1) # bash-4 or to fit
44 else:
45 first, _ = os.path.splitext(path)
46 label = os.path.basename(first)
47
48 # Hack for 'toysh': it's the only shell we call as 'sh'
49 if label == 'sh':
50 label = 'toysh'
51
52 if label == 'osh':
53 # change the second 'osh' to 'osh_ALT' so it's distinct
54 if saw_osh:
55 if OSH_CPP_RE.search(path):
56 label = 'osh-cpp'
57 else:
58 label = 'osh_ALT'
59 saw_osh = True
60
61 elif label == 'ysh':
62 if saw_ysh:
63 if YSH_CPP_RE.search(path):
64 label = 'ysh-cpp'
65 else:
66 label = 'ysh_ALT'
67
68 saw_ysh = True
69
70 elif label == 'oil': # TODO: remove this
71 if saw_oil:
72 if OIL_CPP_RE.search(path):
73 label = 'oil-cpp'
74 else:
75 label = 'oil_ALT'
76
77 saw_oil = True
78
79 shell_pairs.append((label, path))
80 return shell_pairs
81
82
83RANGE_RE = re.compile('(\d+) \s* - \s* (\d+)', re.VERBOSE)
84
85
86def ParseRange(range_str):
87 try:
88 d = int(range_str)
89 return d, d # singleton range
90 except ValueError:
91 m = RANGE_RE.match(range_str)
92 if not m:
93 raise RuntimeError('Invalid range %r' % range_str)
94 b, e = m.groups()
95 return int(b), int(e)
96
97
98class RangePredicate(object):
99 """Zero-based indexing, inclusive ranges."""
100
101 def __init__(self, begin, end):
102 self.begin = begin
103 self.end = end
104
105 def __call__(self, i, case):
106 return self.begin <= i <= self.end
107
108
109class RegexPredicate(object):
110 """Filter by name."""
111
112 def __init__(self, desc_re):
113 self.desc_re = desc_re
114
115 def __call__(self, i, case):
116 return bool(self.desc_re.search(case['desc']))
117
118
119def DefineCommon(p):
120 """Flags shared between sh_spec.py and stateful/harness.py."""
121 p.add_option('-v',
122 '--verbose',
123 dest='verbose',
124 action='store_true',
125 default=False,
126 help='Show details about test failures')
127 p.add_option(
128 '-r',
129 '--range',
130 dest='range',
131 default=None,
132 help='Execute only a given test range, e.g. 5-10, 5-, -10, or 5')
133 p.add_option(
134 '--regex',
135 dest='regex',
136 default=None,
137 help='Execute only tests whose description matches a given regex '
138 '(case-insensitive)')
139 p.add_option('--list',
140 dest='do_list',
141 action='store_true',
142 default=None,
143 help='Just list tests')
144 p.add_option('--oils-failures-allowed',
145 dest='oils_failures_allowed',
146 type='int',
147 default=0,
148 help="Allow this number of Oils failures")
149
150 # Select what shells to run
151 p.add_option('--oils-bin-dir',
152 dest='oils_bin_dir',
153 default=None,
154 help="Directory that osh and ysh live in")
155 p.add_option('--oils-cpp-bin-dir',
156 dest='oils_cpp_bin_dir',
157 default=None,
158 help="Directory that native C++ osh and ysh live in")
159 p.add_option('--ovm-bin-dir',
160 dest='ovm_bin_dir',
161 default=None,
162 help="Directory of the legacy OVM/CPython build")
163 p.add_option(
164 '--compare-shells',
165 dest='compare_shells',
166 action='store_true',
167 help="Compare against shells specified at the top of each file")
168
169
170def DefineStateful(p):
171 p.add_option('--num-retries',
172 dest='num_retries',
173 type='int',
174 default=4,
175 help='Number of retries (for spec/stateful only)')
176 p.add_option('--pexpect-timeout',
177 dest='pexpect_timeout',
178 type='float',
179 default=1.0,
180 help='In seconds')
181 p.add_option(
182 '--results-file',
183 dest='results_file',
184 default=None,
185 help='Write table of results to this file. Default is stdout.')
186 # 24x80 (lines X columns) is the pexpect/ptyprocess default
187 p.add_option('--num-lines',
188 dest='num_lines',
189 type='int',
190 default=24,
191 help='Number of lines to emulate in terminal')
192 p.add_option('--num-columns',
193 dest='num_columns',
194 type='int',
195 default=80,
196 help='Number of columns to emulate in terminal')
197
198
199def DefineShSpec(p):
200 p.add_option('-d',
201 '--details',
202 dest='details',
203 action='store_true',
204 default=False,
205 help='Show details even for successful cases (requires -v)')
206 p.add_option('-t',
207 '--trace',
208 dest='trace',
209 action='store_true',
210 default=False,
211 help='trace execution of shells to diagnose hangs')
212
213 # Execution modes
214 p.add_option('-p',
215 '--print',
216 dest='do_print',
217 action='store_true',
218 default=None,
219 help="Print test code, but don't run it")
220 p.add_option('--print-spec-suite',
221 dest='print_spec_suite',
222 action='store_true',
223 default=None,
224 help="Print suite this file belongs to")
225 p.add_option('--print-table',
226 dest='print_table',
227 action='store_true',
228 default=None,
229 help="Print table of test files")
230 p.add_option('--print-tagged',
231 dest='print_tagged',
232 help="Print spec files tagged with a certain string")
233
234 # Output control
235 p.add_option('--format',
236 dest='format',
237 choices=['ansi', 'html'],
238 default='ansi',
239 help="Output format (default 'ansi')")
240 p.add_option('--stats-file',
241 dest='stats_file',
242 default=None,
243 help="File to write stats to")
244 p.add_option('--tsv-output',
245 dest='tsv_output',
246 default=None,
247 help="Write a TSV log to this file. Subsumes --stats-file.")
248 p.add_option('--stats-template',
249 dest='stats_template',
250 default='',
251 help="Python format string for stats")
252
253 p.add_option('--path-env',
254 dest='path_env',
255 default='',
256 help="The full PATH, for finding binaries used in tests.")
257 p.add_option('--tmp-env',
258 dest='tmp_env',
259 default='',
260 help="A temporary directory that the tests can use.")
261
262 # Notes:
263 # - utf-8 is the Ubuntu default
264 # - this flag has limited usefulness. It may be better to simply export LANG=
265 # in this test case itself.
266 if 0:
267 p.add_option(
268 '--lang-env',
269 dest='lang_env',
270 default='en_US.UTF-8',
271 help="The LANG= setting, which affects various libc functions.")
272 p.add_option('--env-pair',
273 dest='env_pair',
274 default=[],
275 action='append',
276 help='A key=value pair to add to the environment')
277
278 p.add_option('--timeout',
279 dest='timeout',
280 default='',
281 help="Prefix shell invocation with 'timeout N'")
282 p.add_option('--timeout-bin',
283 dest='timeout_bin',
284 default=None,
285 help="Use the smoosh timeout binary at this location.")
286
287 p.add_option('--posix',
288 dest='posix',
289 default=False,
290 action='store_true',
291 help='Pass -o posix to the shell (when applicable)')
292
293 p.add_option('--sh-env-var-name',
294 dest='sh_env_var_name',
295 default='SH',
296 help="Set this environment variable to the path of the shell")
297
298 p.add_option('--pyann-out-dir',
299 dest='pyann_out_dir',
300 default=None,
301 help='Run OSH with PYANN_OUT=$dir/$case_num.json')