| 1 | #!/usr/bin/env python3
|
| 2 | """
|
| 3 | State Machine style tests with pexpect, e.g. for interactive mode.
|
| 4 |
|
| 5 | To invoke this file, run the shell wrapper:
|
| 6 |
|
| 7 | test/stateful.sh all
|
| 8 | """
|
| 9 | from __future__ import print_function
|
| 10 |
|
| 11 | import optparse
|
| 12 | import os
|
| 13 | import pexpect
|
| 14 | import re
|
| 15 | import signal
|
| 16 | import sys
|
| 17 |
|
| 18 | from display import ansi
|
| 19 | from test import spec_lib # Using this for a common interface
|
| 20 |
|
| 21 | log = spec_lib.log
|
| 22 |
|
| 23 |
|
| 24 | def expect_prompt(sh):
|
| 25 | sh.expect(r'.*\$')
|
| 26 |
|
| 27 |
|
| 28 | def get_pid_by_name(name):
|
| 29 | """Return the pid of the process matching `name`."""
|
| 30 | # XXX: make sure this is restricted to subprocesses under us.
|
| 31 | # This could be problematic on the continuous build if many tests are running
|
| 32 | # in parallel.
|
| 33 | output = pexpect.run('pgrep --exact --newest %s' % name)
|
| 34 | #log('pgrep output %r' % output)
|
| 35 | return int(output.split()[-1])
|
| 36 |
|
| 37 |
|
| 38 | def stop_process__hack(name, sig_num=signal.SIGSTOP):
|
| 39 | """Send SIGSTOP to the most recent process matching `name`
|
| 40 |
|
| 41 | Hack in place of sh.sendcontrol('z'), which sends SIGTSTP. Why doesn't OSH
|
| 42 | respond to this, or why don't the child processes respond?
|
| 43 |
|
| 44 | TODO: Fix OSH and get rid of this hack.
|
| 45 | """
|
| 46 | os.kill(get_pid_by_name(name), sig_num)
|
| 47 |
|
| 48 |
|
| 49 | # Mutated by each test file.
|
| 50 | CASES = []
|
| 51 |
|
| 52 |
|
| 53 | def register(skip_shells=None, not_impl_shells=None, needs_dimensions=False):
|
| 54 | skip_shells = skip_shells or []
|
| 55 | not_impl_shells = not_impl_shells or []
|
| 56 |
|
| 57 | def decorator(func):
|
| 58 | CASES.append((func.__doc__, func, skip_shells, not_impl_shells,
|
| 59 | needs_dimensions))
|
| 60 | return func
|
| 61 |
|
| 62 | return decorator
|
| 63 |
|
| 64 |
|
| 65 | class Result(object):
|
| 66 | SKIP = 1
|
| 67 | NI = 2
|
| 68 | OK = 3
|
| 69 | FAIL = 4
|
| 70 |
|
| 71 |
|
| 72 | class TerminalDimensionEnvVars:
|
| 73 | """
|
| 74 | Context manager for setting and unsetting LINES and COLUMNS environment variables.
|
| 75 | """
|
| 76 |
|
| 77 | def __init__(self, lines: int, columns: int):
|
| 78 | self.lines = lines
|
| 79 | self.columns = columns
|
| 80 | self.original_lines = None
|
| 81 | self.original_columns = None
|
| 82 |
|
| 83 | def __enter__(self):
|
| 84 | # Save original values
|
| 85 | self.original_lines = os.environ.get('LINES')
|
| 86 | self.original_columns = os.environ.get('COLUMNS')
|
| 87 |
|
| 88 | # Set new values
|
| 89 | os.environ['LINES'] = str(self.lines)
|
| 90 | os.environ['COLUMNS'] = str(self.columns)
|
| 91 | return self
|
| 92 |
|
| 93 | def __exit__(self, exc_type, exc_val, exc_tb):
|
| 94 | # Restore original values
|
| 95 | if self.original_lines is None:
|
| 96 | del os.environ['LINES']
|
| 97 | else:
|
| 98 | os.environ['LINES'] = self.original_lines
|
| 99 |
|
| 100 | if self.original_columns is None:
|
| 101 | del os.environ['COLUMNS']
|
| 102 | else:
|
| 103 | os.environ['COLUMNS'] = self.original_columns
|
| 104 |
|
| 105 |
|
| 106 | class TestRunner(object):
|
| 107 |
|
| 108 | def __init__(self, num_retries, pexpect_timeout, verbose, num_lines,
|
| 109 | num_columns):
|
| 110 | self.num_retries = num_retries
|
| 111 | self.pexpect_timeout = pexpect_timeout
|
| 112 | self.verbose = verbose
|
| 113 | self.num_lines = num_lines
|
| 114 | self.num_columns = num_columns
|
| 115 |
|
| 116 | def RunOnce(self, shell_path, shell_label, func, test_params={}):
|
| 117 | sh_argv = []
|
| 118 | if shell_label in ('bash', 'osh'):
|
| 119 | sh_argv.extend(['--rcfile', '/dev/null'])
|
| 120 | # Why the heck is --norc different from --rcfile /dev/null in bash??? This
|
| 121 | # makes it so the prompt of the parent shell doesn't leak. Very annoying.
|
| 122 | if shell_label == 'bash':
|
| 123 | sh_argv.append('--norc')
|
| 124 | #print(sh_argv)
|
| 125 |
|
| 126 | # Python 3: encoding required
|
| 127 | sh = pexpect.spawn(
|
| 128 | shell_path,
|
| 129 | sh_argv,
|
| 130 | encoding="utf-8",
|
| 131 | dimensions=(self.num_lines, self.num_columns),
|
| 132 | # Generally don't want local echo of input, it gets confusing fast.
|
| 133 | echo=False,
|
| 134 | timeout=self.pexpect_timeout,
|
| 135 | )
|
| 136 |
|
| 137 | sh.shell_label = shell_label # for tests to use
|
| 138 |
|
| 139 | if self.verbose:
|
| 140 | sh.logfile = sys.stdout
|
| 141 |
|
| 142 | ok = True
|
| 143 | try:
|
| 144 | # Support tests that need extra params (like dimensions), without
|
| 145 | # impacting existing tests
|
| 146 | if len(test_params) > 0:
|
| 147 | func(sh, test_params)
|
| 148 | else:
|
| 149 | func(sh)
|
| 150 | except Exception:
|
| 151 | import traceback
|
| 152 | traceback.print_exc(file=sys.stderr)
|
| 153 | return Result.FAIL
|
| 154 | ok = False
|
| 155 |
|
| 156 | finally:
|
| 157 | sh.close()
|
| 158 |
|
| 159 | if ok:
|
| 160 | return Result.OK
|
| 161 |
|
| 162 | def RunCase(self, shell_path, shell_label, func, test_params={}):
|
| 163 | result = self.RunOnce(shell_path, shell_label, func, test_params)
|
| 164 |
|
| 165 | if result == Result.OK:
|
| 166 | return result, -1 # short circuit for speed
|
| 167 |
|
| 168 | elif result == Result.FAIL:
|
| 169 | num_success = 0
|
| 170 | if self.num_retries:
|
| 171 | log('\tFAILED first time: Retrying 4 times')
|
| 172 | for i in range(self.num_retries):
|
| 173 | log('\tRetry %d of %d', i + 1, self.num_retries)
|
| 174 | result = self.RunOnce(shell_path, shell_label, func,
|
| 175 | test_params)
|
| 176 | if result == Result.OK:
|
| 177 | num_success += 1
|
| 178 | else:
|
| 179 | log('\tFAILED')
|
| 180 |
|
| 181 | if num_success >= 2:
|
| 182 | return Result.OK, num_success
|
| 183 | else:
|
| 184 | return Result.FAIL, num_success
|
| 185 |
|
| 186 | else:
|
| 187 | raise AssertionError(result)
|
| 188 |
|
| 189 | def RunCases(self, cases, case_predicate, shell_pairs, result_table,
|
| 190 | flaky):
|
| 191 | for case_num, (desc, func, skip_shells, not_impl_shells,
|
| 192 | needs_dimensions) in enumerate(cases):
|
| 193 | if not case_predicate(case_num, desc):
|
| 194 | continue
|
| 195 |
|
| 196 | result_row = [case_num]
|
| 197 |
|
| 198 | test_params = {}
|
| 199 | if needs_dimensions:
|
| 200 | test_params['num_lines'] = self.num_lines
|
| 201 | test_params['num_columns'] = self.num_columns
|
| 202 |
|
| 203 | for shell_label, shell_path in shell_pairs:
|
| 204 | skip_str = ''
|
| 205 | if shell_label in skip_shells:
|
| 206 | skip_str = 'SKIP'
|
| 207 | if shell_label in not_impl_shells:
|
| 208 | skip_str = 'N-I'
|
| 209 |
|
| 210 | print()
|
| 211 | print('%s\t%d\t%s\t%s' %
|
| 212 | (skip_str, case_num, shell_label, desc))
|
| 213 | print()
|
| 214 | sys.stdout.flush() # prevent interleaving
|
| 215 |
|
| 216 | if shell_label in skip_shells:
|
| 217 | result_row.append(Result.SKIP)
|
| 218 | flaky[case_num, shell_label] = -1
|
| 219 | continue
|
| 220 |
|
| 221 | # N-I is just like SKIP, but it's displayed differently
|
| 222 | if shell_label in not_impl_shells:
|
| 223 | result_row.append(Result.NI)
|
| 224 | flaky[case_num, shell_label] = -1
|
| 225 | continue
|
| 226 |
|
| 227 | result, retries = self.RunCase(shell_path, shell_label, func,
|
| 228 | test_params)
|
| 229 | flaky[case_num, shell_label] = retries
|
| 230 |
|
| 231 | result_row.append(result)
|
| 232 |
|
| 233 | result_row.append(desc)
|
| 234 | result_table.append(result_row)
|
| 235 |
|
| 236 |
|
| 237 | def PrintResults(shell_pairs, result_table, flaky, num_retries, f):
|
| 238 |
|
| 239 | # Note: In retrospect, it would be better if every process writes a "long"
|
| 240 | # TSV file of results.
|
| 241 | # And then we concatenate them and write the "wide" summary here.
|
| 242 |
|
| 243 | if f.isatty():
|
| 244 | fail_color = ansi.BOLD + ansi.RED
|
| 245 | ok_color = ansi.BOLD + ansi.GREEN
|
| 246 | bold = ansi.BOLD
|
| 247 | reset = ansi.RESET
|
| 248 | else:
|
| 249 | fail_color = ''
|
| 250 | ok_color = ''
|
| 251 | bold = ''
|
| 252 | reset = ''
|
| 253 |
|
| 254 | f.write('\n')
|
| 255 |
|
| 256 | # TODO: Might want an HTML version too
|
| 257 | sh_labels = [shell_label for shell_label, _ in shell_pairs]
|
| 258 |
|
| 259 | f.write(bold)
|
| 260 | f.write('case\t') # case number
|
| 261 | for sh_label in sh_labels:
|
| 262 | f.write(sh_label)
|
| 263 | f.write('\t')
|
| 264 | f.write(reset)
|
| 265 | f.write('\n')
|
| 266 |
|
| 267 | num_failures = 0
|
| 268 |
|
| 269 | for row in result_table:
|
| 270 |
|
| 271 | case_num = row[0]
|
| 272 | desc = row[-1]
|
| 273 |
|
| 274 | f.write('%d\t' % case_num)
|
| 275 |
|
| 276 | num_shells = len(row) - 2
|
| 277 | extra_row = [''] * num_shells
|
| 278 |
|
| 279 | for j, cell in enumerate(row[1:-1]):
|
| 280 | shell_label = sh_labels[j]
|
| 281 |
|
| 282 | num_success = flaky[case_num, shell_label]
|
| 283 | if num_success != -1:
|
| 284 | # the first of 5 failed
|
| 285 | extra_row[j] = '%d/%d ok' % (num_success, num_retries + 1)
|
| 286 |
|
| 287 | if cell == Result.SKIP:
|
| 288 | f.write('SKIP\t')
|
| 289 |
|
| 290 | elif cell == Result.NI:
|
| 291 | f.write('N-I\t')
|
| 292 |
|
| 293 | elif cell == Result.FAIL:
|
| 294 | # Don't count C++ failures right now
|
| 295 | if shell_label != 'osh-cpp':
|
| 296 | log('Ignoring osh-cpp failure: %d %s', case_num, desc)
|
| 297 | num_failures += 1
|
| 298 | f.write('%sFAIL%s\t' % (fail_color, reset))
|
| 299 |
|
| 300 | elif cell == Result.OK:
|
| 301 | f.write('%sok%s\t' % (ok_color, reset))
|
| 302 |
|
| 303 | else:
|
| 304 | raise AssertionError(cell)
|
| 305 |
|
| 306 | f.write(desc)
|
| 307 | f.write('\n')
|
| 308 |
|
| 309 | if any(extra_row):
|
| 310 | for cell in extra_row:
|
| 311 | f.write('\t%s' % cell)
|
| 312 | f.write('\n')
|
| 313 |
|
| 314 | return num_failures
|
| 315 |
|
| 316 |
|
| 317 | def TestStop(exe):
|
| 318 | if 0:
|
| 319 | p = pexpect.spawn('/bin/dash', encoding='utf-8', timeout=2.0)
|
| 320 |
|
| 321 | # Show output
|
| 322 | p.logfile = sys.stdout
|
| 323 | #p.setecho(True)
|
| 324 |
|
| 325 | p.expect(r'.*\$')
|
| 326 | p.sendline('sleep 2')
|
| 327 |
|
| 328 | import time
|
| 329 | time.sleep(0.1)
|
| 330 |
|
| 331 | # Ctrl-C works for the child here
|
| 332 | p.sendcontrol('c')
|
| 333 | p.sendline('echo status=$?')
|
| 334 | p.expect('status=130')
|
| 335 |
|
| 336 | p.close()
|
| 337 |
|
| 338 | return
|
| 339 |
|
| 340 | # Note: pty.fork() calls os.setsid()
|
| 341 | # How does that affect signaling and the process group?
|
| 342 |
|
| 343 | p = pexpect.spawn(exe, encoding='utf-8', timeout=2.0)
|
| 344 |
|
| 345 | # Show output
|
| 346 | p.logfile = sys.stdout
|
| 347 | #p.setecho(True)
|
| 348 |
|
| 349 | p.sendline('sleep 2')
|
| 350 | p.expect('in child')
|
| 351 |
|
| 352 | import time
|
| 353 | time.sleep(0.1)
|
| 354 |
|
| 355 | log('Harness PID %d', os.getpid())
|
| 356 |
|
| 357 | #input()
|
| 358 |
|
| 359 | # Stop it
|
| 360 |
|
| 361 | if 1:
|
| 362 | # Main process gets KeyboardInterrupt
|
| 363 | # hm but child process doesn't get interrupted? why not?
|
| 364 | p.sendcontrol('c')
|
| 365 | if 0: # does NOT work -- why?
|
| 366 | p.sendcontrol('z')
|
| 367 | if 0: # does NOT work
|
| 368 | stop_process__hack('sleep', sig_num=signal.SIGTSTP)
|
| 369 | if 0:
|
| 370 | # WORKS
|
| 371 | stop_process__hack('sleep', sig_num=signal.SIGSTOP)
|
| 372 |
|
| 373 | # These will kill the parent, not the sleep child
|
| 374 | #p.kill(signal.SIGTSTP)
|
| 375 | #p.kill(signal.SIGSTOP)
|
| 376 |
|
| 377 | p.expect('wait =>')
|
| 378 | p.close()
|
| 379 |
|
| 380 |
|
| 381 | def main(argv):
|
| 382 | p = optparse.OptionParser('%s [options] TEST_FILE shell...' % sys.argv[0])
|
| 383 | spec_lib.DefineCommon(p)
|
| 384 | spec_lib.DefineStateful(p)
|
| 385 | opts, argv = p.parse_args(argv)
|
| 386 |
|
| 387 | if len(argv) >= 2 and argv[1] == 'test-stop': # Hack for testing
|
| 388 | TestStop(argv[2])
|
| 389 | return
|
| 390 |
|
| 391 | # List test cases and return
|
| 392 | if opts.do_list:
|
| 393 | for i, (desc, *_) in enumerate(CASES):
|
| 394 | print('%d\t%s' % (i, desc))
|
| 395 | return
|
| 396 |
|
| 397 | shells = argv[1:]
|
| 398 | if not shells:
|
| 399 | raise RuntimeError('Expected shells to run')
|
| 400 |
|
| 401 | shell_pairs = spec_lib.MakeShellPairs(shells)
|
| 402 |
|
| 403 | if opts.range:
|
| 404 | begin, end = spec_lib.ParseRange(opts.range)
|
| 405 | case_predicate = spec_lib.RangePredicate(begin, end)
|
| 406 | elif opts.regex:
|
| 407 | desc_re = re.compile(opts.regex, re.IGNORECASE)
|
| 408 | case_predicate = spec_lib.RegexPredicate(desc_re)
|
| 409 | else:
|
| 410 | case_predicate = lambda i, case: True
|
| 411 |
|
| 412 | if 0:
|
| 413 | print(shell_pairs)
|
| 414 | print(CASES)
|
| 415 |
|
| 416 | result_table = [] # each row is a list
|
| 417 | flaky = {} # (case_num, shell) -> (succeeded, attempted)
|
| 418 |
|
| 419 | r = TestRunner(opts.num_retries, opts.pexpect_timeout, opts.verbose,
|
| 420 | opts.num_lines, opts.num_columns)
|
| 421 | r.RunCases(CASES, case_predicate, shell_pairs, result_table, flaky)
|
| 422 |
|
| 423 | if opts.results_file:
|
| 424 | results_f = open(opts.results_file, 'w')
|
| 425 | else:
|
| 426 | results_f = sys.stdout
|
| 427 | num_failures = PrintResults(shell_pairs, result_table, flaky,
|
| 428 | opts.num_retries, results_f)
|
| 429 |
|
| 430 | results_f.close()
|
| 431 |
|
| 432 | if opts.oils_failures_allowed != num_failures:
|
| 433 | log('%s: Expected %d failures, got %d', sys.argv[0],
|
| 434 | opts.oils_failures_allowed, num_failures)
|
| 435 | return 1
|
| 436 |
|
| 437 | return 0
|
| 438 |
|
| 439 |
|
| 440 | if __name__ == '__main__':
|
| 441 | try:
|
| 442 | sys.exit(main(sys.argv))
|
| 443 | except RuntimeError as e:
|
| 444 | print('FATAL: %s' % e, file=sys.stderr)
|
| 445 | sys.exit(1)
|
| 446 |
|
| 447 | # vim: sw=2
|