| 1 | #!/usr/bin/env python2
|
| 2 | from __future__ import print_function
|
| 3 | """
|
| 4 | sh_spec.py -- Test framework to compare shells.
|
| 5 |
|
| 6 | Assertion help:
|
| 7 | stdout: A single line of expected stdout. Newline is implicit.
|
| 8 | stdout-json: JSON-encoded string. Use for the empty string (no newline),
|
| 9 | for unicode chars, etc.
|
| 10 |
|
| 11 | stderr: Ditto for stderr stream.
|
| 12 | status: Expected shell return code. If not specified, the case must exit 0.
|
| 13 |
|
| 14 | Results:
|
| 15 | PASS - we got the ideal, expected value
|
| 16 | OK - we got a value that was not ideal, but expected
|
| 17 | For OSH this is behavior that was defined to be different?
|
| 18 | N-I - Not implemented (e.g. $''). Assertions still checked (in case it
|
| 19 | starts working)
|
| 20 | BUG - we verified the value of a known bug
|
| 21 | FAIL - we got an unexpected value. If the implementation can't be changed,
|
| 22 | it should be converted to BUG or OK. Otherwise it should be made to
|
| 23 | PASS.
|
| 24 |
|
| 25 | NOTE: The difference between OK and BUG is a matter of judgement. If the ideal
|
| 26 | behavior is a compile time error (code 2), a runtime error is generally OK.
|
| 27 |
|
| 28 | If ALL shells agree on a broken behavior, they are all marked OK (but our
|
| 29 | implementation will be PASS.) But if the behavior is NOT POSIX compliant, then
|
| 30 | it will be a BUG.
|
| 31 |
|
| 32 | If one shell disagrees with others, that is generally a BUG.
|
| 33 |
|
| 34 | Example test case:
|
| 35 |
|
| 36 | #### hello and fail
|
| 37 | echo hello
|
| 38 | echo world
|
| 39 | exit 1
|
| 40 | ## status: 1
|
| 41 | #
|
| 42 | # ignored comment
|
| 43 | #
|
| 44 | ## STDOUT:
|
| 45 | hello
|
| 46 | world
|
| 47 | ## END
|
| 48 |
|
| 49 | """
|
| 50 |
|
| 51 | import collections
|
| 52 | import cgi
|
| 53 | import cStringIO
|
| 54 | import errno
|
| 55 | import json
|
| 56 | import optparse
|
| 57 | import os
|
| 58 | import pprint
|
| 59 | import re
|
| 60 | import shutil
|
| 61 | import subprocess
|
| 62 | import sys
|
| 63 |
|
| 64 | from test import spec_lib
|
| 65 | from doctools import html_head
|
| 66 |
|
| 67 | log = spec_lib.log
|
| 68 |
|
| 69 | # Magic strings for other variants of OSH.
|
| 70 |
|
| 71 | # NOTE: osh_ALT is usually _bin/osh -- the release binary.
|
| 72 | # It would be better to rename these osh-cpython and osh-ovm. Have the concept
|
| 73 | # of a suffix?
|
| 74 |
|
| 75 | OSH_CPYTHON = ('osh', 'osh-dbg')
|
| 76 | OTHER_OSH = ('osh_ALT', )
|
| 77 |
|
| 78 | YSH_CPYTHON = ('ysh', 'ysh-dbg')
|
| 79 | OTHER_YSH = ('oil_ALT', )
|
| 80 |
|
| 81 | # For now, only count the Oils CPython failures. TODO: the spec-cpp job should
|
| 82 | # assert the osh-cpp and ysh-cpp deltas.
|
| 83 | OTHER_OILS = OTHER_OSH + OTHER_YSH + ('osh-cpp', 'ysh-cpp')
|
| 84 |
|
| 85 |
|
| 86 | class ParseError(Exception):
|
| 87 | pass
|
| 88 |
|
| 89 |
|
| 90 | # EXAMPLES:
|
| 91 | ## stdout: foo
|
| 92 | ## stdout-json: ""
|
| 93 | #
|
| 94 | # In other words, it could be (name, value) or (qualifier, name, value)
|
| 95 |
|
| 96 | KEY_VALUE_RE = re.compile(
|
| 97 | r'''
|
| 98 | [#][#] \s+
|
| 99 | # optional prefix with qualifier and shells
|
| 100 | (?: (OK(?:-\d)? | BUG(?:-\d)? | N-I) \s+ ([\w+/]+) \s+ )?
|
| 101 | ([\w\-]+) # key
|
| 102 | :
|
| 103 | \s* (.*) # value
|
| 104 | ''', re.VERBOSE)
|
| 105 |
|
| 106 | END_MULTILINE_RE = re.compile(r'''
|
| 107 | [#][#] \s+ END
|
| 108 | ''', re.VERBOSE)
|
| 109 |
|
| 110 | # Line types
|
| 111 | TEST_CASE_BEGIN = 0 # Starts with ####
|
| 112 | KEY_VALUE = 1 # Metadata
|
| 113 | KEY_VALUE_MULTILINE = 2 # STDOUT STDERR
|
| 114 | END_MULTILINE = 3 # STDOUT STDERR
|
| 115 | PLAIN_LINE = 4 # Uncommented
|
| 116 | EOF = 5
|
| 117 |
|
| 118 | LEX_OUTER = 0 # Ignore blank lines, e.g. for separating cases
|
| 119 | LEX_RAW = 1 # Blank lines are significant
|
| 120 |
|
| 121 |
|
| 122 | class Tokenizer(object):
|
| 123 | """Modal lexer!"""
|
| 124 |
|
| 125 | def __init__(self, f):
|
| 126 | self.f = f
|
| 127 |
|
| 128 | self.cursor = None
|
| 129 | self.line_num = 0
|
| 130 |
|
| 131 | self.next()
|
| 132 |
|
| 133 | def _ClassifyLine(self, line, lex_mode):
|
| 134 | if not line: # empty
|
| 135 | return self.line_num, EOF, ''
|
| 136 |
|
| 137 | if lex_mode == LEX_OUTER and not line.strip():
|
| 138 | return None
|
| 139 |
|
| 140 | if line.startswith('####'):
|
| 141 | desc = line[4:].strip()
|
| 142 | return self.line_num, TEST_CASE_BEGIN, desc
|
| 143 |
|
| 144 | m = KEY_VALUE_RE.match(line)
|
| 145 | if m:
|
| 146 | qualifier, shells, name, value = m.groups()
|
| 147 | # HACK: Expected data should have the newline.
|
| 148 | if name in ('stdout', 'stderr'):
|
| 149 | value += '\n'
|
| 150 |
|
| 151 | if name in ('STDOUT', 'STDERR'):
|
| 152 | token_type = KEY_VALUE_MULTILINE
|
| 153 | else:
|
| 154 | token_type = KEY_VALUE
|
| 155 | return self.line_num, token_type, (qualifier, shells, name, value)
|
| 156 |
|
| 157 | m = END_MULTILINE_RE.match(line)
|
| 158 | if m:
|
| 159 | return self.line_num, END_MULTILINE, None
|
| 160 |
|
| 161 | # If it starts with ##, it should be metadata. This finds some typos.
|
| 162 | if line.startswith('##'):
|
| 163 | raise RuntimeError('Invalid ## line %r' % line)
|
| 164 |
|
| 165 | # TODO: comments should not be ignored in code or STDOUT blocks!
|
| 166 | if line.lstrip().startswith('#'): # Ignore comments
|
| 167 | return None # try again
|
| 168 |
|
| 169 | # Non-empty line that doesn't start with '#'
|
| 170 | # NOTE: We need the original line to test the whitespace sensitive <<-.
|
| 171 | # And we need rstrip because we add newlines back below.
|
| 172 | return self.line_num, PLAIN_LINE, line
|
| 173 |
|
| 174 | def next(self, lex_mode=LEX_OUTER):
|
| 175 | """Raises StopIteration when exhausted."""
|
| 176 | while True:
|
| 177 | line = self.f.readline()
|
| 178 | self.line_num += 1
|
| 179 |
|
| 180 | tok = self._ClassifyLine(line, lex_mode)
|
| 181 | if tok is not None:
|
| 182 | break
|
| 183 |
|
| 184 | self.cursor = tok
|
| 185 | return self.cursor
|
| 186 |
|
| 187 | def peek(self):
|
| 188 | return self.cursor
|
| 189 |
|
| 190 |
|
| 191 | def AddMetadataToCase(case, qualifier, shells, name, value, line_num):
|
| 192 | shells = shells.split('/') # bash/dash/mksh
|
| 193 | for shell in shells:
|
| 194 | if shell not in case:
|
| 195 | case[shell] = {}
|
| 196 |
|
| 197 | # Check a duplicate specification
|
| 198 | name_without_type = re.sub(r'-(json|repr)$', '', name)
|
| 199 | if (name_without_type in case[shell] or
|
| 200 | name_without_type + '-json' in case[shell] or
|
| 201 | name_without_type + '-repr' in case[shell]):
|
| 202 | raise ParseError('Line %d: duplicate spec %r for %r' %
|
| 203 | (line_num, name, shell))
|
| 204 |
|
| 205 | # Check inconsistent qualifier
|
| 206 | if 'qualifier' in case[shell] and qualifier != case[shell]['qualifier']:
|
| 207 | raise ParseError(
|
| 208 | 'Line %d: inconsistent qualifier %r is specified for %r, '
|
| 209 | 'but %r was previously specified. ' %
|
| 210 | (line_num, qualifier, shell, case[shell]['qualifier']))
|
| 211 |
|
| 212 | case[shell][name] = value
|
| 213 | case[shell]['qualifier'] = qualifier
|
| 214 |
|
| 215 |
|
| 216 | # Format of a test script.
|
| 217 | #
|
| 218 | # -- Code is either literal lines, or a commented out code: value.
|
| 219 | # code = PLAIN_LINE*
|
| 220 | # | '## code:' VALUE
|
| 221 | #
|
| 222 | # -- Key value pairs can be single- or multi-line
|
| 223 | # key_value = '##' KEY ':' VALUE
|
| 224 | # | KEY_VALUE_MULTILINE PLAIN_LINE* END_MULTILINE
|
| 225 | #
|
| 226 | # -- Description, then key-value pairs surrounding code.
|
| 227 | # test_case = '####' DESC
|
| 228 | # key_value*
|
| 229 | # code
|
| 230 | # key_value*
|
| 231 | #
|
| 232 | # -- Should be a blank line after each test case. Leading comments and code
|
| 233 | # -- are OK.
|
| 234 | #
|
| 235 | # test_file =
|
| 236 | # key_value* -- file level metadata
|
| 237 | # (test_case '\n')*
|
| 238 |
|
| 239 |
|
| 240 | def ParseKeyValue(tokens, case):
|
| 241 | """Parse commented-out metadata in a test case.
|
| 242 |
|
| 243 | The metadata must be contiguous.
|
| 244 |
|
| 245 | Args:
|
| 246 | tokens: Tokenizer
|
| 247 | case: dictionary to add to
|
| 248 | """
|
| 249 | while True:
|
| 250 | line_num, kind, item = tokens.peek()
|
| 251 |
|
| 252 | if kind == KEY_VALUE_MULTILINE:
|
| 253 | qualifier, shells, name, empty_value = item
|
| 254 | if empty_value:
|
| 255 | raise ParseError(
|
| 256 | 'Line %d: got value %r for %r, but the value should be on the '
|
| 257 | 'following lines' % (line_num, empty_value, name))
|
| 258 |
|
| 259 | value_lines = []
|
| 260 | while True:
|
| 261 | tokens.next(lex_mode=LEX_RAW) # empty lines aren't skipped
|
| 262 | _, kind2, item2 = tokens.peek()
|
| 263 | if kind2 != PLAIN_LINE:
|
| 264 | break
|
| 265 | value_lines.append(item2)
|
| 266 |
|
| 267 | value = ''.join(value_lines)
|
| 268 |
|
| 269 | name = name.lower() # STDOUT -> stdout
|
| 270 | if qualifier:
|
| 271 | AddMetadataToCase(case, qualifier, shells, name, value,
|
| 272 | line_num)
|
| 273 | else:
|
| 274 | case[name] = value
|
| 275 |
|
| 276 | # END token is optional.
|
| 277 | if kind2 == END_MULTILINE:
|
| 278 | tokens.next()
|
| 279 |
|
| 280 | elif kind == KEY_VALUE:
|
| 281 | qualifier, shells, name, value = item
|
| 282 |
|
| 283 | if qualifier:
|
| 284 | AddMetadataToCase(case, qualifier, shells, name, value,
|
| 285 | line_num)
|
| 286 | else:
|
| 287 | case[name] = value
|
| 288 |
|
| 289 | tokens.next()
|
| 290 |
|
| 291 | else: # Unknown token type
|
| 292 | break
|
| 293 |
|
| 294 |
|
| 295 | def ParseCodeLines(tokens, case):
|
| 296 | """Parse uncommented code in a test case."""
|
| 297 | _, kind, item = tokens.peek()
|
| 298 | if kind != PLAIN_LINE:
|
| 299 | raise ParseError('Expected a line of code (got %r, %r)' % (kind, item))
|
| 300 | code_lines = []
|
| 301 | while True:
|
| 302 | _, kind, item = tokens.peek()
|
| 303 | if kind != PLAIN_LINE:
|
| 304 | case['code'] = ''.join(code_lines)
|
| 305 | return
|
| 306 | code_lines.append(item)
|
| 307 | tokens.next(lex_mode=LEX_RAW)
|
| 308 |
|
| 309 |
|
| 310 | def ParseTestCase(tokens):
|
| 311 | """Parse a single test case and return it.
|
| 312 |
|
| 313 | If at EOF, return None.
|
| 314 | """
|
| 315 | line_num, kind, item = tokens.peek()
|
| 316 | if kind == EOF:
|
| 317 | return None
|
| 318 |
|
| 319 | if kind != TEST_CASE_BEGIN:
|
| 320 | raise RuntimeError("line %d: Expected TEST_CASE_BEGIN, got %r" %
|
| 321 | (line_num, [kind, item]))
|
| 322 |
|
| 323 | tokens.next()
|
| 324 |
|
| 325 | case = {'desc': item, 'line_num': line_num}
|
| 326 |
|
| 327 | ParseKeyValue(tokens, case)
|
| 328 |
|
| 329 | # For broken code
|
| 330 | if 'code' in case: # Got it through a key value pair
|
| 331 | return case
|
| 332 |
|
| 333 | ParseCodeLines(tokens, case)
|
| 334 | ParseKeyValue(tokens, case)
|
| 335 |
|
| 336 | return case
|
| 337 |
|
| 338 |
|
| 339 | _META_FIELDS = [
|
| 340 | 'our_shell',
|
| 341 | 'compare_shells',
|
| 342 | 'suite',
|
| 343 | 'tags',
|
| 344 | 'oils_failures_allowed',
|
| 345 | 'oils_cpp_failures_allowed',
|
| 346 | 'legacy_tmp_dir',
|
| 347 | ]
|
| 348 |
|
| 349 |
|
| 350 | def ParseTestFile(test_file, tokens):
|
| 351 | """
|
| 352 | test_file: Only for error message
|
| 353 | """
|
| 354 | file_metadata = {}
|
| 355 | test_cases = []
|
| 356 |
|
| 357 | try:
|
| 358 | # Skip over the header. Setup code can go here, although would we have to
|
| 359 | # execute it on every case?
|
| 360 | while True:
|
| 361 | line_num, kind, item = tokens.peek()
|
| 362 | if kind != KEY_VALUE:
|
| 363 | break
|
| 364 |
|
| 365 | qualifier, shells, name, value = item
|
| 366 | if qualifier is not None:
|
| 367 | raise RuntimeError('Invalid qualifier in spec file metadata')
|
| 368 | if shells is not None:
|
| 369 | raise RuntimeError('Invalid shells in spec file metadata')
|
| 370 |
|
| 371 | file_metadata[name] = value
|
| 372 |
|
| 373 | tokens.next()
|
| 374 |
|
| 375 | while True: # Loop over cases
|
| 376 | test_case = ParseTestCase(tokens)
|
| 377 | if test_case is None:
|
| 378 | break
|
| 379 | test_cases.append(test_case)
|
| 380 |
|
| 381 | except StopIteration:
|
| 382 | raise RuntimeError('Unexpected EOF parsing test cases')
|
| 383 |
|
| 384 | for name in file_metadata:
|
| 385 | if name not in _META_FIELDS:
|
| 386 | raise RuntimeError('Invalid file metadata %r in %r' %
|
| 387 | (name, test_file))
|
| 388 |
|
| 389 | return file_metadata, test_cases
|
| 390 |
|
| 391 |
|
| 392 | def CreateStringAssertion(d, key, assertions, qualifier=False):
|
| 393 | found = False
|
| 394 |
|
| 395 | exp = d.get(key)
|
| 396 | if exp is not None:
|
| 397 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
| 398 | assertions.append(a)
|
| 399 | found = True
|
| 400 |
|
| 401 | exp_json = d.get(key + '-json')
|
| 402 | if exp_json is not None:
|
| 403 | exp = json.loads(exp_json, encoding='utf-8')
|
| 404 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
| 405 | assertions.append(a)
|
| 406 | found = True
|
| 407 |
|
| 408 | # For testing invalid unicode
|
| 409 | exp_repr = d.get(key + '-repr')
|
| 410 | if exp_repr is not None:
|
| 411 | exp = eval(exp_repr)
|
| 412 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
| 413 | assertions.append(a)
|
| 414 | found = True
|
| 415 |
|
| 416 | return found
|
| 417 |
|
| 418 |
|
| 419 | def CreateIntAssertion(d, key, assertions, qualifier=False):
|
| 420 | exp = d.get(key) # expected
|
| 421 | if exp is not None:
|
| 422 | # For now, turn it into int
|
| 423 | a = EqualAssertion(key, int(exp), qualifier=qualifier)
|
| 424 | assertions.append(a)
|
| 425 | return True
|
| 426 | return False
|
| 427 |
|
| 428 |
|
| 429 | def CreateAssertions(case, sh_label):
|
| 430 | """Given a raw test case and a shell label, create EqualAssertion instances
|
| 431 | to run."""
|
| 432 | assertions = []
|
| 433 |
|
| 434 | # Whether we found assertions
|
| 435 | stdout = False
|
| 436 | stderr = False
|
| 437 | status = False
|
| 438 |
|
| 439 | # So the assertion are exactly the same for osh and osh_ALT
|
| 440 |
|
| 441 | if sh_label.startswith('osh'):
|
| 442 | case_sh = 'osh'
|
| 443 | elif sh_label.startswith('bash'):
|
| 444 | case_sh = 'bash'
|
| 445 | else:
|
| 446 | case_sh = sh_label
|
| 447 |
|
| 448 | if case_sh in case:
|
| 449 | q = case[case_sh]['qualifier']
|
| 450 | if CreateStringAssertion(case[case_sh],
|
| 451 | 'stdout',
|
| 452 | assertions,
|
| 453 | qualifier=q):
|
| 454 | stdout = True
|
| 455 | if CreateStringAssertion(case[case_sh],
|
| 456 | 'stderr',
|
| 457 | assertions,
|
| 458 | qualifier=q):
|
| 459 | stderr = True
|
| 460 | if CreateIntAssertion(case[case_sh], 'status', assertions,
|
| 461 | qualifier=q):
|
| 462 | status = True
|
| 463 |
|
| 464 | if not stdout:
|
| 465 | CreateStringAssertion(case, 'stdout', assertions)
|
| 466 | if not stderr:
|
| 467 | CreateStringAssertion(case, 'stderr', assertions)
|
| 468 | if not status:
|
| 469 | if 'status' in case:
|
| 470 | CreateIntAssertion(case, 'status', assertions)
|
| 471 | else:
|
| 472 | # If the user didn't specify a 'status' assertion, assert that the exit
|
| 473 | # code is 0.
|
| 474 | a = EqualAssertion('status', 0)
|
| 475 | assertions.append(a)
|
| 476 |
|
| 477 | no_traceback = SubstringAssertion('stderr', 'Traceback (most recent')
|
| 478 | assertions.append(no_traceback)
|
| 479 |
|
| 480 | #print 'SHELL', shell
|
| 481 | #pprint.pprint(case)
|
| 482 | #print(assertions)
|
| 483 | return assertions
|
| 484 |
|
| 485 |
|
| 486 | class Result(object):
|
| 487 | """Result of an stdout/stderr/status assertion or of a (case, shell) cell.
|
| 488 |
|
| 489 | Order is important: the result of a cell is the minimum of the results of
|
| 490 | each assertion.
|
| 491 | """
|
| 492 | TIMEOUT = 0 # ONLY a cell result, not an assertion result
|
| 493 | FAIL = 1
|
| 494 | BUG = 2
|
| 495 | BUG_2 = 3
|
| 496 | NI = 4
|
| 497 | OK = 5
|
| 498 | OK_2 = 6
|
| 499 | OK_3 = 7
|
| 500 | OK_4 = 8
|
| 501 | PASS = 9
|
| 502 |
|
| 503 | length = 10 # for loops
|
| 504 |
|
| 505 |
|
| 506 | def QualifierToResult(qualifier):
|
| 507 | # type: (str) -> Result
|
| 508 | if qualifier == 'BUG': # equal, but known bad
|
| 509 | return Result.BUG
|
| 510 | if qualifier == 'BUG-2':
|
| 511 | return Result.BUG_2
|
| 512 |
|
| 513 | if qualifier == 'N-I': # equal, and known UNIMPLEMENTED
|
| 514 | return Result.NI
|
| 515 |
|
| 516 | if qualifier == 'OK': # equal, but ok (not ideal)
|
| 517 | return Result.OK
|
| 518 | if qualifier == 'OK-2':
|
| 519 | return Result.OK_2
|
| 520 | if qualifier == 'OK-3':
|
| 521 | return Result.OK_3
|
| 522 | if qualifier == 'OK-4':
|
| 523 | return Result.OK_4
|
| 524 |
|
| 525 | return Result.PASS # ideal behavior
|
| 526 |
|
| 527 |
|
| 528 | class EqualAssertion(object):
|
| 529 | """Check that two values are equal."""
|
| 530 |
|
| 531 | def __init__(self, key, expected, qualifier=None):
|
| 532 | self.key = key
|
| 533 | self.expected = expected # expected value
|
| 534 | self.qualifier = qualifier # whether this was a special case?
|
| 535 |
|
| 536 | def __repr__(self):
|
| 537 | return '<EqualAssertion %s == %r>' % (self.key, self.expected)
|
| 538 |
|
| 539 | def Check(self, shell, record):
|
| 540 | actual = record[self.key]
|
| 541 | if actual != self.expected:
|
| 542 | if len(str(self.expected)) < 40:
|
| 543 | msg = '[%s %s] Expected %r, got %r' % (shell, self.key,
|
| 544 | self.expected, actual)
|
| 545 | else:
|
| 546 | msg = '''
|
| 547 | [%s %s]
|
| 548 | Expected %r
|
| 549 | Got %r
|
| 550 | ''' % (shell, self.key, self.expected, actual)
|
| 551 |
|
| 552 | # TODO: Make this better and add a flag for it.
|
| 553 | if 0:
|
| 554 | import difflib
|
| 555 | for line in difflib.unified_diff(self.expected,
|
| 556 | actual,
|
| 557 | fromfile='expected',
|
| 558 | tofile='actual'):
|
| 559 | print(repr(line))
|
| 560 |
|
| 561 | return Result.FAIL, msg
|
| 562 |
|
| 563 | return QualifierToResult(self.qualifier), ''
|
| 564 |
|
| 565 |
|
| 566 | class SubstringAssertion(object):
|
| 567 | """Check that a string like stderr doesn't have a substring."""
|
| 568 |
|
| 569 | def __init__(self, key, substring):
|
| 570 | self.key = key
|
| 571 | self.substring = substring
|
| 572 |
|
| 573 | def __repr__(self):
|
| 574 | return '<SubstringAssertion %s == %r>' % (self.key, self.substring)
|
| 575 |
|
| 576 | def Check(self, shell, record):
|
| 577 | actual = record[self.key]
|
| 578 | if self.substring in actual:
|
| 579 | msg = '[%s %s] Found %r' % (shell, self.key, self.substring)
|
| 580 | return Result.FAIL, msg
|
| 581 | return Result.PASS, ''
|
| 582 |
|
| 583 |
|
| 584 | class Stats(object):
|
| 585 |
|
| 586 | def __init__(self, num_cases, sh_labels):
|
| 587 | self.counters = collections.defaultdict(int)
|
| 588 | c = self.counters
|
| 589 | c['num_cases'] = num_cases
|
| 590 | c['oils_num_passed'] = 0
|
| 591 | c['oils_num_failed'] = 0
|
| 592 | c['oils_cpp_num_failed'] = 0
|
| 593 | # Number of osh_ALT results that differed from osh.
|
| 594 | c['oils_ALT_delta'] = 0
|
| 595 |
|
| 596 | self.by_shell = {}
|
| 597 | for sh in sh_labels:
|
| 598 | self.by_shell[sh] = collections.defaultdict(int)
|
| 599 | self.nonzero_results = collections.defaultdict(int)
|
| 600 |
|
| 601 | self.tsv_rows = []
|
| 602 |
|
| 603 | def Inc(self, counter_name):
|
| 604 | self.counters[counter_name] += 1
|
| 605 |
|
| 606 | def Get(self, counter_name):
|
| 607 | return self.counters[counter_name]
|
| 608 |
|
| 609 | def Set(self, counter_name, val):
|
| 610 | self.counters[counter_name] = val
|
| 611 |
|
| 612 | def ReportCell(self, case_num, cell_result, sh_label):
|
| 613 | self.tsv_rows.append(
|
| 614 | (str(case_num), sh_label, TEXT_CELLS[cell_result]))
|
| 615 |
|
| 616 | self.by_shell[sh_label][cell_result] += 1
|
| 617 | self.nonzero_results[cell_result] += 1
|
| 618 |
|
| 619 | c = self.counters
|
| 620 | if cell_result == Result.TIMEOUT:
|
| 621 | c['num_timeout'] += 1
|
| 622 | elif cell_result == Result.FAIL:
|
| 623 | # Special logic: don't count osh_ALT because its failures will be
|
| 624 | # counted in the delta.
|
| 625 | if sh_label not in OTHER_OILS:
|
| 626 | c['num_failed'] += 1
|
| 627 |
|
| 628 | if sh_label in OSH_CPYTHON + YSH_CPYTHON:
|
| 629 | c['oils_num_failed'] += 1
|
| 630 |
|
| 631 | if sh_label in ('osh-cpp', 'ysh-cpp'):
|
| 632 | c['oils_cpp_num_failed'] += 1
|
| 633 | elif cell_result in (Result.BUG, Result.BUG_2):
|
| 634 | c['num_bug'] += 1
|
| 635 | elif cell_result == Result.NI:
|
| 636 | c['num_ni'] += 1
|
| 637 | elif cell_result in (Result.OK, Result.OK_2, Result.OK_3, Result.OK_4):
|
| 638 | c['num_ok'] += 1
|
| 639 | elif cell_result == Result.PASS:
|
| 640 | c['num_passed'] += 1
|
| 641 | if sh_label in OSH_CPYTHON + YSH_CPYTHON:
|
| 642 | c['oils_num_passed'] += 1
|
| 643 | else:
|
| 644 | raise AssertionError()
|
| 645 |
|
| 646 | def WriteTsv(self, f):
|
| 647 | f.write('case\tshell\tresult\n')
|
| 648 | for row in self.tsv_rows:
|
| 649 | f.write('\t'.join(row))
|
| 650 | f.write('\n')
|
| 651 |
|
| 652 |
|
| 653 | PIPE = subprocess.PIPE
|
| 654 |
|
| 655 |
|
| 656 | def _TimedOut(status):
|
| 657 | # timeout 1s sleep 5 ==> status 124
|
| 658 | # timeout -s KILL 1s sleep 5 ==> status 137
|
| 659 | # the latter is more robust
|
| 660 | #
|
| 661 | # SIGKILL == -9
|
| 662 | #return status in (124, 137)
|
| 663 | return status == -9
|
| 664 |
|
| 665 |
|
| 666 | def _PrepareCaseTempDir(case_tmp_dir, legacy_tmp_dir=False):
|
| 667 | # Clean up after the previous run. (The previous run doesn't clean
|
| 668 | # up after itself, because it's useful to manually inspect the
|
| 669 | # state.)
|
| 670 | try:
|
| 671 | shutil.rmtree(case_tmp_dir)
|
| 672 | except OSError as e:
|
| 673 | #log('Error cleaning up %r: %s', case_tmp_dir, e)
|
| 674 | pass
|
| 675 |
|
| 676 | try:
|
| 677 | os.makedirs(case_tmp_dir)
|
| 678 | except OSError as e:
|
| 679 | if e.errno != errno.EEXIST:
|
| 680 | raise
|
| 681 |
|
| 682 | # Some tests assume _tmp exists
|
| 683 | if legacy_tmp_dir:
|
| 684 | try:
|
| 685 | os.mkdir(os.path.join(case_tmp_dir, '_tmp'))
|
| 686 | except OSError as e:
|
| 687 | if e.errno != errno.EEXIST:
|
| 688 | raise
|
| 689 |
|
| 690 |
|
| 691 | def RunCases(cases,
|
| 692 | case_predicate,
|
| 693 | shells,
|
| 694 | env,
|
| 695 | out,
|
| 696 | opts,
|
| 697 | legacy_tmp_dir=False):
|
| 698 | """Run a list of test 'cases' for all 'shells' and write output to
|
| 699 | 'out'."""
|
| 700 | if opts.trace:
|
| 701 | for _, sh in shells:
|
| 702 | log('\tshell: %s', sh)
|
| 703 | print('\twhich $SH: ', end='', file=sys.stderr)
|
| 704 | subprocess.call(['which', sh])
|
| 705 |
|
| 706 | #pprint.pprint(cases)
|
| 707 |
|
| 708 | if (isinstance(case_predicate, spec_lib.RangePredicate) and
|
| 709 | (case_predicate.begin > (len(cases) - 1))):
|
| 710 | raise RuntimeError(
|
| 711 | "valid case indexes are from 0 to %s. given range: %s-%s" %
|
| 712 | ((len(cases) - 1), case_predicate.begin, case_predicate.end))
|
| 713 |
|
| 714 | sh_labels = [sh_label for sh_label, _ in shells]
|
| 715 |
|
| 716 | out.WriteHeader(sh_labels)
|
| 717 | stats = Stats(len(cases), sh_labels)
|
| 718 |
|
| 719 | # Make an environment for each shell. $SH is the path to the shell, so we
|
| 720 | # can test flags, etc.
|
| 721 | sh_env = []
|
| 722 | for _, sh_path in shells:
|
| 723 | e = dict(env)
|
| 724 | e[opts.sh_env_var_name] = sh_path
|
| 725 | sh_env.append(e)
|
| 726 |
|
| 727 | # Determine which one (if any) is osh-cpython, for comparison against other
|
| 728 | # shells.
|
| 729 | osh_cpython_index = -1
|
| 730 | for i, (sh_label, _) in enumerate(shells):
|
| 731 | if sh_label in OSH_CPYTHON:
|
| 732 | osh_cpython_index = i
|
| 733 | break
|
| 734 |
|
| 735 | timeout_dir = os.path.abspath('_tmp/spec/timeouts')
|
| 736 | try:
|
| 737 | shutil.rmtree(timeout_dir)
|
| 738 | os.mkdir(timeout_dir)
|
| 739 | except OSError:
|
| 740 | pass
|
| 741 |
|
| 742 | # Now run each case, and print a table.
|
| 743 | for i, case in enumerate(cases):
|
| 744 | line_num = case['line_num']
|
| 745 | desc = case['desc']
|
| 746 | code = case['code']
|
| 747 |
|
| 748 | if opts.trace:
|
| 749 | log('case %d: %s', i, desc)
|
| 750 |
|
| 751 | if not case_predicate(i, case):
|
| 752 | stats.Inc('num_skipped')
|
| 753 | continue
|
| 754 |
|
| 755 | if opts.do_print:
|
| 756 | print('#### %s' % case['desc'])
|
| 757 | print(case['code'])
|
| 758 | print()
|
| 759 | continue
|
| 760 |
|
| 761 | stats.Inc('num_cases_run')
|
| 762 |
|
| 763 | result_row = []
|
| 764 |
|
| 765 | for shell_index, (sh_label, sh_path) in enumerate(shells):
|
| 766 | timeout_file = os.path.join(timeout_dir, '%02d-%s' % (i, sh_label))
|
| 767 | if opts.timeout:
|
| 768 | if opts.timeout_bin:
|
| 769 | # This is what smoosh itself uses. See smoosh/tests/shell_tests.sh
|
| 770 | # QUIRK: interval can only be a whole number
|
| 771 | argv = [
|
| 772 | opts.timeout_bin,
|
| 773 | '-t',
|
| 774 | opts.timeout,
|
| 775 | # Somehow I'm not able to get this timeout file working? I think
|
| 776 | # it has a bug when using stdin. It waits for the background
|
| 777 | # process too.
|
| 778 |
|
| 779 | #'-i', '1',
|
| 780 | #'-l', timeout_file
|
| 781 | ]
|
| 782 | else:
|
| 783 | # This kills hanging tests properly, but somehow they fail with code
|
| 784 | # -9?
|
| 785 | argv = ['timeout', '-s', 'KILL', opts.timeout + 's']
|
| 786 |
|
| 787 | # s suffix for seconds
|
| 788 | #argv = ['timeout', opts.timeout + 's']
|
| 789 | else:
|
| 790 | argv = []
|
| 791 | argv.append(sh_path)
|
| 792 |
|
| 793 | # dash doesn't support -o posix
|
| 794 | if opts.posix and sh_label != 'dash':
|
| 795 | argv.extend(['-o', 'posix'])
|
| 796 |
|
| 797 | if opts.trace:
|
| 798 | log('\targv: %s', ' '.join(argv))
|
| 799 |
|
| 800 | case_env = sh_env[shell_index]
|
| 801 |
|
| 802 | # Unique dir for every test case and shell
|
| 803 | tmp_base = os.path.normpath(opts.tmp_env) # no . or ..
|
| 804 | case_tmp_dir = os.path.join(tmp_base, '%02d-%s' % (i, sh_label))
|
| 805 |
|
| 806 | _PrepareCaseTempDir(case_tmp_dir, legacy_tmp_dir=legacy_tmp_dir)
|
| 807 |
|
| 808 | case_env['TMP'] = case_tmp_dir
|
| 809 |
|
| 810 | if opts.pyann_out_dir:
|
| 811 | case_env = dict(case_env)
|
| 812 | case_env['PYANN_OUT'] = os.path.join(opts.pyann_out_dir,
|
| 813 | '%d.json' % i)
|
| 814 |
|
| 815 | try:
|
| 816 | p = subprocess.Popen(argv,
|
| 817 | env=case_env,
|
| 818 | cwd=case_tmp_dir,
|
| 819 | stdin=PIPE,
|
| 820 | stdout=PIPE,
|
| 821 | stderr=PIPE)
|
| 822 | except OSError as e:
|
| 823 | print('Error running %r: %s' % (sh_path, e), file=sys.stderr)
|
| 824 | sys.exit(1)
|
| 825 |
|
| 826 | p.stdin.write(code)
|
| 827 |
|
| 828 | actual = {}
|
| 829 | actual['stdout'], actual['stderr'] = p.communicate()
|
| 830 | actual['status'] = p.wait()
|
| 831 |
|
| 832 | if opts.timeout_bin and os.path.exists(timeout_file):
|
| 833 | cell_result = Result.TIMEOUT
|
| 834 | elif not opts.timeout_bin and _TimedOut(actual['status']):
|
| 835 | cell_result = Result.TIMEOUT
|
| 836 | else:
|
| 837 | messages = []
|
| 838 | cell_result = Result.PASS
|
| 839 |
|
| 840 | # TODO: Warn about no assertions? Well it will always test the error
|
| 841 | # code.
|
| 842 | assertions = CreateAssertions(case, sh_label)
|
| 843 | for a in assertions:
|
| 844 | result, msg = a.Check(sh_label, actual)
|
| 845 | # The minimum one wins.
|
| 846 | # If any failed, then the result is FAIL.
|
| 847 | # If any are OK, but none are FAIL, the result is OK.
|
| 848 | cell_result = min(cell_result, result)
|
| 849 | if msg:
|
| 850 | messages.append(msg)
|
| 851 |
|
| 852 | if cell_result != Result.PASS or opts.details:
|
| 853 | d = (i, sh_label, actual['stdout'], actual['stderr'],
|
| 854 | messages)
|
| 855 | out.AddDetails(d)
|
| 856 |
|
| 857 | result_row.append(cell_result)
|
| 858 |
|
| 859 | stats.ReportCell(i, cell_result, sh_label)
|
| 860 |
|
| 861 | if sh_label in OTHER_OSH:
|
| 862 | # This is only an error if we tried to run ANY OSH.
|
| 863 | if osh_cpython_index == -1:
|
| 864 | raise RuntimeError(
|
| 865 | "Couldn't determine index of osh-cpython")
|
| 866 |
|
| 867 | other_result = result_row[shell_index]
|
| 868 | cpython_result = result_row[osh_cpython_index]
|
| 869 | if other_result != cpython_result:
|
| 870 | stats.Inc('oils_ALT_delta')
|
| 871 |
|
| 872 | out.WriteRow(i, line_num, result_row, desc)
|
| 873 |
|
| 874 | return stats
|
| 875 |
|
| 876 |
|
| 877 | # ANSI color constants
|
| 878 | _RESET = '\033[0;0m'
|
| 879 | _BOLD = '\033[1m'
|
| 880 |
|
| 881 | _RED = '\033[31m'
|
| 882 | _GREEN = '\033[32m'
|
| 883 | _YELLOW = '\033[33m'
|
| 884 | _BLUE = '\033[34m'
|
| 885 | _PURPLE = '\033[35m'
|
| 886 | _CYAN = '\033[36m'
|
| 887 | #_WHITE = '\033[37m'
|
| 888 |
|
| 889 | TEXT_CELLS = {
|
| 890 | Result.TIMEOUT: 'TIME',
|
| 891 | Result.FAIL: 'FAIL',
|
| 892 | Result.BUG: 'BUG',
|
| 893 | Result.BUG_2: 'BUG-2',
|
| 894 | Result.NI: 'N-I',
|
| 895 | Result.OK: 'ok',
|
| 896 | Result.OK_2: 'ok-2',
|
| 897 | Result.OK_3: 'ok-3',
|
| 898 | Result.OK_4: 'ok-4',
|
| 899 | Result.PASS: 'pass',
|
| 900 | }
|
| 901 |
|
| 902 | ANSI_COLORS = {
|
| 903 | Result.TIMEOUT: _PURPLE,
|
| 904 | Result.FAIL: _RED,
|
| 905 | Result.BUG: _YELLOW,
|
| 906 | Result.BUG_2: _BLUE,
|
| 907 | Result.NI: _YELLOW,
|
| 908 | Result.OK: _YELLOW,
|
| 909 | Result.OK_2: _BLUE,
|
| 910 | Result.OK_3: _CYAN,
|
| 911 | Result.OK_4: _PURPLE,
|
| 912 | Result.PASS: _GREEN,
|
| 913 | }
|
| 914 |
|
| 915 |
|
| 916 | def _AnsiCells():
|
| 917 | lookup = {}
|
| 918 | for i in xrange(Result.length):
|
| 919 | lookup[i] = ''.join([ANSI_COLORS[i], _BOLD, TEXT_CELLS[i], _RESET])
|
| 920 | return lookup
|
| 921 |
|
| 922 |
|
| 923 | ANSI_CELLS = _AnsiCells()
|
| 924 |
|
| 925 | HTML_CELLS = {
|
| 926 | Result.TIMEOUT: '<td class="timeout">TIME',
|
| 927 | Result.FAIL: '<td class="fail">FAIL',
|
| 928 | Result.BUG: '<td class="bug">BUG',
|
| 929 | Result.BUG_2: '<td class="bug-2">BUG-2',
|
| 930 | Result.NI: '<td class="n-i">N-I',
|
| 931 | Result.OK: '<td class="ok">ok',
|
| 932 | Result.OK_2: '<td class="ok-2">ok-2',
|
| 933 | Result.OK_3: '<td class="ok-3">ok-3',
|
| 934 | Result.OK_4: '<td class="ok-4">ok-4',
|
| 935 | Result.PASS: '<td class="pass">pass',
|
| 936 | }
|
| 937 |
|
| 938 |
|
| 939 | def _ValidUtf8String(s):
|
| 940 | """Return an arbitrary string as a readable utf-8 string.
|
| 941 |
|
| 942 | We output utf-8 to either HTML or the console. If we get invalid
|
| 943 | utf-8 as stdout/stderr (which is very possible), then show the ASCII
|
| 944 | repr().
|
| 945 | """
|
| 946 | try:
|
| 947 | s.decode('utf-8')
|
| 948 | return s # it decoded OK
|
| 949 | except UnicodeDecodeError:
|
| 950 | return repr(s) # ASCII representation
|
| 951 |
|
| 952 |
|
| 953 | class Output(object):
|
| 954 |
|
| 955 | def __init__(self, f, verbose):
|
| 956 | self.f = f
|
| 957 | self.verbose = verbose
|
| 958 | self.details = []
|
| 959 |
|
| 960 | def BeginCases(self, test_file):
|
| 961 | pass
|
| 962 |
|
| 963 | def WriteHeader(self, sh_labels):
|
| 964 | pass
|
| 965 |
|
| 966 | def WriteRow(self, i, line_num, row, desc):
|
| 967 | pass
|
| 968 |
|
| 969 | def EndCases(self, sh_labels, stats):
|
| 970 | pass
|
| 971 |
|
| 972 | def AddDetails(self, entry):
|
| 973 | self.details.append(entry)
|
| 974 |
|
| 975 | # Helper function
|
| 976 | def _WriteDetailsAsText(self, details):
|
| 977 | for case_index, shell, stdout, stderr, messages in details:
|
| 978 | print('case: %d' % case_index, file=self.f)
|
| 979 | for m in messages:
|
| 980 | print(m, file=self.f)
|
| 981 |
|
| 982 | # Assume the terminal can show utf-8, but we don't want random binary.
|
| 983 | print('%s stdout:' % shell, file=self.f)
|
| 984 | print(_ValidUtf8String(stdout), file=self.f)
|
| 985 |
|
| 986 | print('%s stderr:' % shell, file=self.f)
|
| 987 | print(_ValidUtf8String(stderr), file=self.f)
|
| 988 |
|
| 989 | print('', file=self.f)
|
| 990 |
|
| 991 |
|
| 992 | class TeeOutput(object):
|
| 993 | """For multiple outputs in one run, e.g. HTML and TSV.
|
| 994 |
|
| 995 | UNUSED
|
| 996 | """
|
| 997 |
|
| 998 | def __init__(self, outs):
|
| 999 | self.outs = outs
|
| 1000 |
|
| 1001 | def BeginCases(self, test_file):
|
| 1002 | for out in self.outs:
|
| 1003 | out.BeginCases(test_file)
|
| 1004 |
|
| 1005 | def WriteHeader(self, sh_labels):
|
| 1006 | for out in self.outs:
|
| 1007 | out.WriteHeader(sh_labels)
|
| 1008 |
|
| 1009 | def WriteRow(self, i, line_num, row, desc):
|
| 1010 | for out in self.outs:
|
| 1011 | out.WriteRow(i, line_num, row, desc)
|
| 1012 |
|
| 1013 | def EndCases(self, sh_labels, stats):
|
| 1014 | for out in self.outs:
|
| 1015 | out.EndCases(sh_labels, stats)
|
| 1016 |
|
| 1017 | def AddDetails(self, entry):
|
| 1018 | for out in self.outs:
|
| 1019 | out.AddDetails(entry)
|
| 1020 |
|
| 1021 |
|
| 1022 | class TsvOutput(Output):
|
| 1023 | """Write a plain-text TSV file.
|
| 1024 |
|
| 1025 | UNUSED since we are outputting LONG format with --tsv-output.
|
| 1026 | """
|
| 1027 |
|
| 1028 | def WriteHeader(self, sh_labels):
|
| 1029 | self.f.write('case\tline\t') # case number and line number
|
| 1030 | for sh_label in sh_labels:
|
| 1031 | self.f.write(sh_label)
|
| 1032 | self.f.write('\t')
|
| 1033 | self.f.write('\n')
|
| 1034 |
|
| 1035 | def WriteRow(self, i, line_num, row, desc):
|
| 1036 | self.f.write('%3d\t%3d\t' % (i, line_num))
|
| 1037 |
|
| 1038 | for result in row:
|
| 1039 | c = TEXT_CELLS[result]
|
| 1040 | self.f.write(c)
|
| 1041 | self.f.write('\t')
|
| 1042 |
|
| 1043 | # note: 'desc' could use TSV8, but just ignore it for now
|
| 1044 | #self.f.write(desc)
|
| 1045 | self.f.write('\n')
|
| 1046 |
|
| 1047 |
|
| 1048 | class AnsiOutput(Output):
|
| 1049 |
|
| 1050 | def BeginCases(self, test_file):
|
| 1051 | self.f.write('%s\n' % test_file)
|
| 1052 |
|
| 1053 | def WriteHeader(self, sh_labels):
|
| 1054 | self.f.write(_BOLD)
|
| 1055 | self.f.write('case\tline\t') # case number and line number
|
| 1056 | for sh_label in sh_labels:
|
| 1057 | self.f.write(sh_label)
|
| 1058 | self.f.write('\t')
|
| 1059 | self.f.write(_RESET)
|
| 1060 | self.f.write('\n')
|
| 1061 |
|
| 1062 | def WriteRow(self, i, line_num, row, desc):
|
| 1063 | self.f.write('%3d\t%3d\t' % (i, line_num))
|
| 1064 |
|
| 1065 | for result in row:
|
| 1066 | c = ANSI_CELLS[result]
|
| 1067 | self.f.write(c)
|
| 1068 | self.f.write('\t')
|
| 1069 |
|
| 1070 | self.f.write(desc)
|
| 1071 | self.f.write('\n')
|
| 1072 |
|
| 1073 | if self.verbose:
|
| 1074 | self._WriteDetailsAsText(self.details)
|
| 1075 | self.details = []
|
| 1076 |
|
| 1077 | def _WriteShellSummary(self, sh_labels, stats):
|
| 1078 | if len(stats.nonzero_results) <= 1: # Skip trivial summaries
|
| 1079 | return
|
| 1080 |
|
| 1081 | # Reiterate header
|
| 1082 | self.f.write(_BOLD)
|
| 1083 | self.f.write('\t\t')
|
| 1084 | for sh_label in sh_labels:
|
| 1085 | self.f.write(sh_label)
|
| 1086 | self.f.write('\t')
|
| 1087 | self.f.write(_RESET)
|
| 1088 | self.f.write('\n')
|
| 1089 |
|
| 1090 | # Write totals by cell.
|
| 1091 | for result in sorted(stats.nonzero_results, reverse=True):
|
| 1092 | self.f.write('\t%s' % ANSI_CELLS[result])
|
| 1093 | for sh_label in sh_labels:
|
| 1094 | self.f.write('\t%d' % stats.by_shell[sh_label][result])
|
| 1095 | self.f.write('\n')
|
| 1096 |
|
| 1097 | # The bottom row is all the same, but it helps readability.
|
| 1098 | self.f.write('\ttotal')
|
| 1099 | for sh_label in sh_labels:
|
| 1100 | self.f.write('\t%d' % stats.counters['num_cases_run'])
|
| 1101 | self.f.write('\n')
|
| 1102 | self.f.write('\n')
|
| 1103 |
|
| 1104 | def EndCases(self, sh_labels, stats):
|
| 1105 | print()
|
| 1106 | self._WriteShellSummary(sh_labels, stats)
|
| 1107 |
|
| 1108 |
|
| 1109 | class HtmlOutput(Output):
|
| 1110 |
|
| 1111 | def __init__(self, f, verbose, spec_name, sh_labels, cases):
|
| 1112 | Output.__init__(self, f, verbose)
|
| 1113 | self.spec_name = spec_name
|
| 1114 | self.sh_labels = sh_labels # saved from header
|
| 1115 | self.cases = cases # for linking to code
|
| 1116 | self.row_html = [] # buffered
|
| 1117 |
|
| 1118 | def _SourceLink(self, line_num, desc):
|
| 1119 | return '<a href="%s.test.html#L%d">%s</a>' % (self.spec_name, line_num,
|
| 1120 | cgi.escape(desc))
|
| 1121 |
|
| 1122 | def BeginCases(self, test_file):
|
| 1123 | css_urls = ['../../../web/base.css', '../../../web/spec-tests.css']
|
| 1124 | title = '%s: spec test case results' % self.spec_name
|
| 1125 | html_head.Write(self.f, title, css_urls=css_urls)
|
| 1126 |
|
| 1127 | self.f.write('''\
|
| 1128 | <body class="width60">
|
| 1129 | <p id="home-link">
|
| 1130 | <a href=".">spec test index</a>
|
| 1131 | /
|
| 1132 | <a href="/">oils.pub</a>
|
| 1133 | </p>
|
| 1134 | <h1>Results for %s</h1>
|
| 1135 | <table>
|
| 1136 | ''' % test_file)
|
| 1137 |
|
| 1138 | def _WriteShellSummary(self, sh_labels, stats):
|
| 1139 | # NOTE: This table has multiple <thead>, which seems OK.
|
| 1140 | self.f.write('''
|
| 1141 | <thead>
|
| 1142 | <tr class="table-header">
|
| 1143 | ''')
|
| 1144 |
|
| 1145 | columns = ['status'] + sh_labels + ['']
|
| 1146 | for c in columns:
|
| 1147 | self.f.write('<td>%s</td>' % c)
|
| 1148 |
|
| 1149 | self.f.write('''
|
| 1150 | </tr>
|
| 1151 | </thead>
|
| 1152 | ''')
|
| 1153 |
|
| 1154 | # Write totals by cell.
|
| 1155 | for result in sorted(stats.nonzero_results, reverse=True):
|
| 1156 | self.f.write('<tr>')
|
| 1157 |
|
| 1158 | self.f.write(HTML_CELLS[result])
|
| 1159 | self.f.write('</td> ')
|
| 1160 |
|
| 1161 | for sh_label in sh_labels:
|
| 1162 | self.f.write('<td>%d</td>' % stats.by_shell[sh_label][result])
|
| 1163 |
|
| 1164 | self.f.write('<td></td>')
|
| 1165 | self.f.write('</tr>\n')
|
| 1166 |
|
| 1167 | # The bottom row is all the same, but it helps readability.
|
| 1168 | self.f.write('<tr>')
|
| 1169 | self.f.write('<td>total</td>')
|
| 1170 | for sh_label in sh_labels:
|
| 1171 | self.f.write('<td>%d</td>' % stats.counters['num_cases_run'])
|
| 1172 | self.f.write('<td></td>')
|
| 1173 | self.f.write('</tr>\n')
|
| 1174 |
|
| 1175 | # Blank row for space.
|
| 1176 | self.f.write('<tr>')
|
| 1177 | for i in xrange(len(sh_labels) + 2):
|
| 1178 | self.f.write('<td style="height: 2em"></td>')
|
| 1179 | self.f.write('</tr>\n')
|
| 1180 |
|
| 1181 | def WriteHeader(self, sh_labels):
|
| 1182 | f = cStringIO.StringIO()
|
| 1183 |
|
| 1184 | f.write('''
|
| 1185 | <thead>
|
| 1186 | <tr class="table-header">
|
| 1187 | ''')
|
| 1188 |
|
| 1189 | columns = ['case'] + sh_labels
|
| 1190 | for c in columns:
|
| 1191 | f.write('<td>%s</td>' % c)
|
| 1192 | f.write('<td class="case-desc">description</td>')
|
| 1193 |
|
| 1194 | f.write('''
|
| 1195 | </tr>
|
| 1196 | </thead>
|
| 1197 | ''')
|
| 1198 |
|
| 1199 | self.row_html.append(f.getvalue())
|
| 1200 |
|
| 1201 | def WriteRow(self, i, line_num, row, desc):
|
| 1202 | f = cStringIO.StringIO()
|
| 1203 | f.write('<tr>')
|
| 1204 | f.write('<td>%3d</td>' % i)
|
| 1205 |
|
| 1206 | show_details = False
|
| 1207 |
|
| 1208 | for result in row:
|
| 1209 | c = HTML_CELLS[result]
|
| 1210 | if result not in (Result.PASS, Result.TIMEOUT): # nothing to show
|
| 1211 | show_details = True
|
| 1212 |
|
| 1213 | f.write(c)
|
| 1214 | f.write('</td>')
|
| 1215 | f.write('\t')
|
| 1216 |
|
| 1217 | f.write('<td class="case-desc">')
|
| 1218 | f.write(self._SourceLink(line_num, desc))
|
| 1219 | f.write('</td>')
|
| 1220 | f.write('</tr>\n')
|
| 1221 |
|
| 1222 | # Show row with details link.
|
| 1223 | if show_details:
|
| 1224 | f.write('<tr>')
|
| 1225 | f.write('<td class="details-row"></td>') # for the number
|
| 1226 |
|
| 1227 | for col_index, result in enumerate(row):
|
| 1228 | f.write('<td class="details-row">')
|
| 1229 | if result != Result.PASS:
|
| 1230 | sh_label = self.sh_labels[col_index]
|
| 1231 | f.write('<a href="#details-%s-%s">details</a>' %
|
| 1232 | (i, sh_label))
|
| 1233 | f.write('</td>')
|
| 1234 |
|
| 1235 | f.write('<td class="details-row"></td>') # for the description
|
| 1236 | f.write('</tr>\n')
|
| 1237 |
|
| 1238 | self.row_html.append(f.getvalue()) # buffer it
|
| 1239 |
|
| 1240 | def _WriteStats(self, stats):
|
| 1241 | self.f.write('%(num_passed)d passed, %(num_ok)d OK, '
|
| 1242 | '%(num_ni)d not implemented, %(num_bug)d BUG, '
|
| 1243 | '%(num_failed)d failed, %(num_timeout)d timeouts, '
|
| 1244 | '%(num_skipped)d cases skipped\n' % stats.counters)
|
| 1245 |
|
| 1246 | def EndCases(self, sh_labels, stats):
|
| 1247 | self._WriteShellSummary(sh_labels, stats)
|
| 1248 |
|
| 1249 | # Write all the buffered rows
|
| 1250 | for h in self.row_html:
|
| 1251 | self.f.write(h)
|
| 1252 |
|
| 1253 | self.f.write('</table>\n')
|
| 1254 | self.f.write('<pre>')
|
| 1255 | self._WriteStats(stats)
|
| 1256 | if stats.Get('oils_num_failed'):
|
| 1257 | self.f.write('%(oils_num_failed)d failed under osh\n' %
|
| 1258 | stats.counters)
|
| 1259 | self.f.write('</pre>')
|
| 1260 |
|
| 1261 | if self.details:
|
| 1262 | self._WriteDetails()
|
| 1263 |
|
| 1264 | self.f.write('</body></html>')
|
| 1265 |
|
| 1266 | def _WriteDetails(self):
|
| 1267 | self.f.write("<h2>Details on runs that didn't PASS</h2>")
|
| 1268 | self.f.write('<table id="details">')
|
| 1269 |
|
| 1270 | for case_index, sh_label, stdout, stderr, messages in self.details:
|
| 1271 | self.f.write('<tr>')
|
| 1272 | self.f.write('<td><a name="details-%s-%s"></a><b>%s</b></td>' %
|
| 1273 | (case_index, sh_label, sh_label))
|
| 1274 |
|
| 1275 | self.f.write('<td>')
|
| 1276 |
|
| 1277 | # Write description and link to the code
|
| 1278 | case = self.cases[case_index]
|
| 1279 | line_num = case['line_num']
|
| 1280 | desc = case['desc']
|
| 1281 | self.f.write('%d ' % case_index)
|
| 1282 | self.f.write(self._SourceLink(line_num, desc))
|
| 1283 | self.f.write('<br/><br/>\n')
|
| 1284 |
|
| 1285 | for m in messages:
|
| 1286 | self.f.write('<span class="assertion">%s</span><br/>\n' %
|
| 1287 | cgi.escape(m))
|
| 1288 | if messages:
|
| 1289 | self.f.write('<br/>\n')
|
| 1290 |
|
| 1291 | def _WriteRaw(s):
|
| 1292 | self.f.write('<pre>')
|
| 1293 |
|
| 1294 | # stdout might contain invalid utf-8; make it valid;
|
| 1295 | valid_utf8 = _ValidUtf8String(s)
|
| 1296 |
|
| 1297 | self.f.write(cgi.escape(valid_utf8))
|
| 1298 | self.f.write('</pre>')
|
| 1299 |
|
| 1300 | self.f.write('<i>stdout:</i> <br/>\n')
|
| 1301 | _WriteRaw(stdout)
|
| 1302 |
|
| 1303 | self.f.write('<i>stderr:</i> <br/>\n')
|
| 1304 | _WriteRaw(stderr)
|
| 1305 |
|
| 1306 | self.f.write('</td>')
|
| 1307 | self.f.write('</tr>')
|
| 1308 |
|
| 1309 | self.f.write('</table>')
|
| 1310 |
|
| 1311 |
|
| 1312 | def MakeTestEnv(opts):
|
| 1313 | if not opts.tmp_env:
|
| 1314 | raise RuntimeError('--tmp-env required')
|
| 1315 | if not opts.path_env:
|
| 1316 | raise RuntimeError('--path-env required')
|
| 1317 | env = {
|
| 1318 | 'PATH': opts.path_env,
|
| 1319 | #'LANG': opts.lang_env,
|
| 1320 | }
|
| 1321 | for p in opts.env_pair:
|
| 1322 | name, value = p.split('=', 1)
|
| 1323 | env[name] = value
|
| 1324 |
|
| 1325 | return env
|
| 1326 |
|
| 1327 |
|
| 1328 | def _DefaultSuite(spec_name):
|
| 1329 | if spec_name.startswith('ysh-'):
|
| 1330 | suite = 'ysh'
|
| 1331 | elif spec_name.startswith('hay'): # hay.test.sh is ysh
|
| 1332 | suite = 'ysh'
|
| 1333 |
|
| 1334 | elif spec_name.startswith('tea-'):
|
| 1335 | suite = 'tea'
|
| 1336 | else:
|
| 1337 | suite = 'osh'
|
| 1338 |
|
| 1339 | return suite
|
| 1340 |
|
| 1341 |
|
| 1342 | def ParseTestList(test_files):
|
| 1343 | for test_file in test_files:
|
| 1344 | with open(test_file) as f:
|
| 1345 | tokens = Tokenizer(f)
|
| 1346 | try:
|
| 1347 | file_metadata, cases = ParseTestFile(test_file, tokens)
|
| 1348 | except RuntimeError as e:
|
| 1349 | log('ERROR in %r', test_file)
|
| 1350 | raise
|
| 1351 | except ParseError as e:
|
| 1352 | log('PARSE ERROR in %r', test_file)
|
| 1353 | raise
|
| 1354 |
|
| 1355 | tmp = os.path.basename(test_file)
|
| 1356 | spec_name = tmp.split('.')[0] # foo.test.sh -> foo
|
| 1357 |
|
| 1358 | suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
|
| 1359 |
|
| 1360 | tmp = file_metadata.get('tags')
|
| 1361 | tags = tmp.split() if tmp else []
|
| 1362 |
|
| 1363 | # Don't need compare_shells, etc. to decide what to run
|
| 1364 |
|
| 1365 | row = {'spec_name': spec_name, 'suite': suite, 'tags': tags}
|
| 1366 | #print(row)
|
| 1367 | yield row
|
| 1368 |
|
| 1369 |
|
| 1370 | def main(argv):
|
| 1371 | # First check if bash is polluting the environment. Tests rely on the
|
| 1372 | # environment.
|
| 1373 | v = os.getenv('RANDOM')
|
| 1374 | if v is not None:
|
| 1375 | raise AssertionError('got $RANDOM = %s' % v)
|
| 1376 | v = os.getenv('PPID')
|
| 1377 | if v is not None:
|
| 1378 | raise AssertionError('got $PPID = %s' % v)
|
| 1379 |
|
| 1380 | p = optparse.OptionParser('%s [options] TEST_FILE shell...' % sys.argv[0])
|
| 1381 | spec_lib.DefineCommon(p)
|
| 1382 | spec_lib.DefineShSpec(p)
|
| 1383 | opts, argv = p.parse_args(argv)
|
| 1384 |
|
| 1385 | # --print-tagged to figure out what to run
|
| 1386 | if opts.print_tagged:
|
| 1387 | to_find = opts.print_tagged
|
| 1388 | for row in ParseTestList(argv[1:]):
|
| 1389 | if to_find in row['tags']:
|
| 1390 | print(row['spec_name'])
|
| 1391 | return 0
|
| 1392 |
|
| 1393 | # --print-table to figure out what to run
|
| 1394 | if opts.print_table:
|
| 1395 | for row in ParseTestList(argv[1:]):
|
| 1396 | print('%(suite)s\t%(spec_name)s' % row)
|
| 1397 | #print(row)
|
| 1398 | return 0
|
| 1399 |
|
| 1400 | #
|
| 1401 | # Now deal with a single file
|
| 1402 | #
|
| 1403 |
|
| 1404 | try:
|
| 1405 | test_file = argv[1]
|
| 1406 | except IndexError:
|
| 1407 | p.print_usage()
|
| 1408 | return 1
|
| 1409 |
|
| 1410 | with open(test_file) as f:
|
| 1411 | tokens = Tokenizer(f)
|
| 1412 | file_metadata, cases = ParseTestFile(test_file, tokens)
|
| 1413 |
|
| 1414 | # List test cases and return
|
| 1415 | if opts.do_list:
|
| 1416 | for i, case in enumerate(cases):
|
| 1417 | if opts.verbose: # print the raw dictionary for debugging
|
| 1418 | print(pprint.pformat(case))
|
| 1419 | else:
|
| 1420 | print('%d\t%s' % (i, case['desc']))
|
| 1421 | return 0
|
| 1422 |
|
| 1423 | # for test/spec-cpp.sh
|
| 1424 | if opts.print_spec_suite:
|
| 1425 | tmp = os.path.basename(test_file)
|
| 1426 | spec_name = tmp.split('.')[0] # foo.test.sh -> foo
|
| 1427 |
|
| 1428 | suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
|
| 1429 | print(suite)
|
| 1430 | return 0
|
| 1431 |
|
| 1432 | if opts.verbose:
|
| 1433 | for k, v in file_metadata.items():
|
| 1434 | print('\t%-20s: %s' % (k, v), file=sys.stderr)
|
| 1435 | print('', file=sys.stderr)
|
| 1436 |
|
| 1437 | if opts.oils_bin_dir: # to find OSH and YSH
|
| 1438 | shells = []
|
| 1439 |
|
| 1440 | if opts.compare_shells:
|
| 1441 | comp = file_metadata.get('compare_shells')
|
| 1442 | # Compare 'compare_shells' and Python
|
| 1443 | shells.extend(comp.split() if comp else [])
|
| 1444 |
|
| 1445 | # Always run with the Python version
|
| 1446 | our_shell = file_metadata.get('our_shell', 'osh') # default is OSH
|
| 1447 | if our_shell != '-':
|
| 1448 | shells.append(os.path.join(opts.oils_bin_dir, our_shell))
|
| 1449 |
|
| 1450 | # Legacy OVM/CPython build
|
| 1451 | if opts.ovm_bin_dir:
|
| 1452 | shells.append(os.path.join(opts.ovm_bin_dir, our_shell))
|
| 1453 |
|
| 1454 | # New C++ build
|
| 1455 | if opts.oils_cpp_bin_dir:
|
| 1456 | shells.append(os.path.join(opts.oils_cpp_bin_dir, our_shell))
|
| 1457 |
|
| 1458 | # Overwrite it when --oils-bin-dir is set
|
| 1459 | # It's no longer a flag
|
| 1460 | opts.oils_failures_allowed = \
|
| 1461 | int(file_metadata.get('oils_failures_allowed', 0))
|
| 1462 |
|
| 1463 | else:
|
| 1464 | # To compare arbitrary shells
|
| 1465 | shells = argv[2:]
|
| 1466 |
|
| 1467 | shell_pairs = spec_lib.MakeShellPairs(shells)
|
| 1468 |
|
| 1469 | if opts.range:
|
| 1470 | begin, end = spec_lib.ParseRange(opts.range)
|
| 1471 | case_predicate = spec_lib.RangePredicate(begin, end)
|
| 1472 | elif opts.regex:
|
| 1473 | desc_re = re.compile(opts.regex, re.IGNORECASE)
|
| 1474 | case_predicate = spec_lib.RegexPredicate(desc_re)
|
| 1475 | else:
|
| 1476 | case_predicate = lambda i, case: True
|
| 1477 |
|
| 1478 | out_f = sys.stderr if opts.do_print else sys.stdout
|
| 1479 |
|
| 1480 | # Set up output style. Also see asdl/format.py
|
| 1481 | if opts.format == 'ansi':
|
| 1482 | out = AnsiOutput(out_f, opts.verbose)
|
| 1483 |
|
| 1484 | elif opts.format == 'html':
|
| 1485 | spec_name = os.path.basename(test_file)
|
| 1486 | spec_name = spec_name.split('.')[0]
|
| 1487 |
|
| 1488 | sh_labels = [label for label, _ in shell_pairs]
|
| 1489 |
|
| 1490 | out = HtmlOutput(out_f, opts.verbose, spec_name, sh_labels, cases)
|
| 1491 |
|
| 1492 | else:
|
| 1493 | raise AssertionError()
|
| 1494 |
|
| 1495 | out.BeginCases(os.path.basename(test_file))
|
| 1496 |
|
| 1497 | env = MakeTestEnv(opts)
|
| 1498 | stats = RunCases(cases,
|
| 1499 | case_predicate,
|
| 1500 | shell_pairs,
|
| 1501 | env,
|
| 1502 | out,
|
| 1503 | opts,
|
| 1504 | legacy_tmp_dir=bool(file_metadata.get('legacy_tmp_dir')))
|
| 1505 |
|
| 1506 | out.EndCases([sh_label for sh_label, _ in shell_pairs], stats)
|
| 1507 |
|
| 1508 | if opts.tsv_output:
|
| 1509 | with open(opts.tsv_output, 'w') as f:
|
| 1510 | stats.WriteTsv(f)
|
| 1511 |
|
| 1512 | # TODO: Could --stats-{file,template} be a separate awk step on .tsv files?
|
| 1513 | stats.Set('oils_failures_allowed', opts.oils_failures_allowed)
|
| 1514 |
|
| 1515 | # If it's not set separately for C++, we default to the allowed number
|
| 1516 | # above
|
| 1517 | x = int(
|
| 1518 | file_metadata.get('oils_cpp_failures_allowed',
|
| 1519 | opts.oils_failures_allowed))
|
| 1520 | stats.Set('oils_cpp_failures_allowed', x)
|
| 1521 |
|
| 1522 | if opts.stats_file:
|
| 1523 | with open(opts.stats_file, 'w') as f:
|
| 1524 | f.write(opts.stats_template % stats.counters)
|
| 1525 | f.write('\n') # bash 'read' requires a newline
|
| 1526 |
|
| 1527 | # spec/smoke.test.sh -> smoke
|
| 1528 | test_name = os.path.basename(test_file).split('.')[0]
|
| 1529 |
|
| 1530 | return _SuccessOrFailure(test_name, stats)
|
| 1531 |
|
| 1532 |
|
| 1533 | def _SuccessOrFailure(test_name, stats):
|
| 1534 | allowed = stats.Get('oils_failures_allowed')
|
| 1535 | allowed_cpp = stats.Get('oils_cpp_failures_allowed')
|
| 1536 |
|
| 1537 | all_count = stats.Get('num_failed')
|
| 1538 | oils_count = stats.Get('oils_num_failed')
|
| 1539 | oils_cpp_count = stats.Get('oils_cpp_num_failed')
|
| 1540 |
|
| 1541 | errors = []
|
| 1542 | if oils_count != allowed:
|
| 1543 | errors.append('Got %d Oils failures, but %d are allowed' %
|
| 1544 | (oils_count, allowed))
|
| 1545 | else:
|
| 1546 | if allowed != 0:
|
| 1547 | log('%s: note: Got %d allowed Oils failures', test_name, allowed)
|
| 1548 |
|
| 1549 | # TODO: remove special case for 0
|
| 1550 | if oils_cpp_count != 0:
|
| 1551 | if oils_cpp_count != allowed_cpp:
|
| 1552 | errors.append('Got %d Oils C++ failures, but %d are allowed' %
|
| 1553 | (oils_cpp_count, allowed_cpp))
|
| 1554 | else:
|
| 1555 | if allowed_cpp != 0:
|
| 1556 | log('%s: note: Got %d allowed Oils C++ failures', test_name,
|
| 1557 | allowed_cpp)
|
| 1558 |
|
| 1559 | if all_count != allowed:
|
| 1560 | errors.append('Got %d total failures, but %d are allowed' %
|
| 1561 | (all_count, allowed))
|
| 1562 |
|
| 1563 | if errors:
|
| 1564 | for msg in errors:
|
| 1565 | log('%s: FATAL: %s', test_name, msg)
|
| 1566 | return 1
|
| 1567 |
|
| 1568 | return 0
|
| 1569 |
|
| 1570 |
|
| 1571 | if __name__ == '__main__':
|
| 1572 | try:
|
| 1573 | sys.exit(main(sys.argv))
|
| 1574 | except KeyboardInterrupt as e:
|
| 1575 | print('%s: interrupted with Ctrl-C' % sys.argv[0], file=sys.stderr)
|
| 1576 | sys.exit(1)
|
| 1577 | except RuntimeError as e:
|
| 1578 | print('FATAL: %s' % e, file=sys.stderr)
|
| 1579 | sys.exit(1)
|