X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=utils%2Flit%2Flit%2FTestRunner.py;h=24075ff94048e4667001db56bb1dff03ee732948;hb=47f0e3f434e2e43f951c3a826c40906cb15b7285;hp=71882b76f8b95f819eb794630d8ed9313487b32b;hpb=b1a464279623768a3d04ff6726c99dce35d2f360;p=oota-llvm.git diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py index 71882b76f8b..24075ff9404 100644 --- a/utils/lit/lit/TestRunner.py +++ b/utils/lit/lit/TestRunner.py @@ -1,14 +1,13 @@ +from __future__ import absolute_import import os, signal, subprocess, sys -import StringIO - -import ShUtil -import Test -import Util - +import re import platform import tempfile -import re +import lit.ShUtil as ShUtil +import lit.Test as Test +import lit.util +from lit.util import to_bytes, to_string class InternalShellError(Exception): def __init__(self, command, message): @@ -23,51 +22,60 @@ kUseCloseFDs = not kIsWindows # Use temporary files to replace /dev/null on Windows. kAvoidDevNull = kIsWindows -def executeCommand(command, cwd=None, env=None): - # Close extra file handles on UNIX (on Windows this cannot be done while - # also redirecting input). - close_fds = not kIsWindows +class ShellEnvironment(object): - p = subprocess.Popen(command, cwd=cwd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, close_fds=close_fds) - out,err = p.communicate() - exitCode = p.wait() + """Mutable shell environment containing things like CWD and env vars. - # Detect Ctrl-C in subprocess. - if exitCode == -signal.SIGINT: - raise KeyboardInterrupt + Environment variables are not implemented, but cwd tracking is. + """ - return out, err, exitCode + def __init__(self, cwd, env): + self.cwd = cwd + self.env = dict(env) -def executeShCmd(cmd, cfg, cwd, results): +def executeShCmd(cmd, shenv, results): if isinstance(cmd, ShUtil.Seq): if cmd.op == ';': - res = executeShCmd(cmd.lhs, cfg, cwd, results) - return executeShCmd(cmd.rhs, cfg, cwd, results) + res = executeShCmd(cmd.lhs, shenv, results) + return executeShCmd(cmd.rhs, shenv, results) if cmd.op == '&': - raise NotImplementedError,"unsupported test command: '&'" + raise InternalShellError(cmd,"unsupported shell operator: '&'") if cmd.op == '||': - res = executeShCmd(cmd.lhs, cfg, cwd, results) + res = executeShCmd(cmd.lhs, shenv, results) if res != 0: - res = executeShCmd(cmd.rhs, cfg, cwd, results) + res = executeShCmd(cmd.rhs, shenv, results) return res + if cmd.op == '&&': - res = executeShCmd(cmd.lhs, cfg, cwd, results) + res = executeShCmd(cmd.lhs, shenv, results) if res is None: return res if res == 0: - res = executeShCmd(cmd.rhs, cfg, cwd, results) + res = executeShCmd(cmd.rhs, shenv, results) return res - raise ValueError,'Unknown shell command: %r' % cmd.op - + raise ValueError('Unknown shell command: %r' % cmd.op) assert isinstance(cmd, ShUtil.Pipeline) + + # Handle shell builtins first. + if cmd.commands[0].args[0] == 'cd': + if len(cmd.commands) != 1: + raise ValueError("'cd' cannot be part of a pipeline") + if len(cmd.commands[0].args) != 2: + raise ValueError("'cd' supports only one argument") + newdir = cmd.commands[0].args[1] + # Update the cwd in the parent environment. + if os.path.isabs(newdir): + shenv.cwd = newdir + else: + shenv.cwd = os.path.join(shenv.cwd, newdir) + # The cd builtin always succeeds. If the directory does not exist, the + # following Popen calls will fail instead. + return 0 + procs = [] input = subprocess.PIPE stderrTempFiles = [] @@ -77,7 +85,24 @@ def executeShCmd(cmd, cfg, cwd, results): # output. This is null until we have seen some output using # stderr. for i,j in enumerate(cmd.commands): - # Apply the redirections, we use (N,) as a sentinal to indicate stdin, + # Reference the global environment by default. + cmd_shenv = shenv + if j.args[0] == 'env': + # Create a copy of the global environment and modify it for this one + # command. There might be multiple envs in a pipeline: + # env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s + cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env) + arg_idx = 1 + for arg_idx, arg in enumerate(j.args[1:]): + # Partition the string into KEY=VALUE. + key, eq, val = arg.partition('=') + # Stop if there was no equals. + if eq == '': + break + cmd_shenv.env[key] = val + j.args = j.args[arg_idx+1:] + + # Apply the redirections, we use (N,) as a sentinel to indicate stdin, # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or # from a file are represented with a list [file, mode, file-object] # where file-object is initially None. @@ -98,7 +123,7 @@ def executeShCmd(cmd, cfg, cwd, results): elif r[0] == ('<',): redirects[0] = [r[1], 'r', None] else: - raise NotImplementedError,"Unsupported redirect: %r" % (r,) + raise InternalShellError(j,"Unsupported redirect: %r" % (r,)) # Map from the final redirections to something subprocess can handle. final_redirects = [] @@ -107,21 +132,23 @@ def executeShCmd(cmd, cfg, cwd, results): result = input elif r == (1,): if index == 0: - raise NotImplementedError,"Unsupported redirect for stdin" + raise InternalShellError(j,"Unsupported redirect for stdin") elif index == 1: result = subprocess.PIPE else: result = subprocess.STDOUT elif r == (2,): if index != 2: - raise NotImplementedError,"Unsupported redirect on stdout" + raise InternalShellError(j,"Unsupported redirect on stdout") result = subprocess.PIPE else: if r[2] is None: if kAvoidDevNull and r[0] == '/dev/null': r[2] = tempfile.TemporaryFile(mode=r[1]) else: - r[2] = open(r[0], r[1]) + # Make sure relative paths are relative to the cwd. + redir_filename = os.path.join(cmd_shenv.cwd, r[0]) + r[2] = open(redir_filename, r[1]) # Workaround a Win32 and/or subprocess bug when appending. # # FIXME: Actually, this is probably an instance of PR6753. @@ -151,8 +178,15 @@ def executeShCmd(cmd, cfg, cwd, results): # Resolve the executable path ourselves. args = list(j.args) - args[0] = Util.which(args[0], cfg.environment['PATH']) - if not args[0]: + executable = None + # For paths relative to cwd, use the cwd of the shell environment. + if args[0].startswith('.'): + exe_in_cwd = os.path.join(cmd_shenv.cwd, args[0]) + if os.path.isfile(exe_in_cwd): + executable = exe_in_cwd + if not executable: + executable = lit.util.which(args[0], cmd_shenv.env['PATH']) + if not executable: raise InternalShellError(j, '%r: command not found' % j.args[0]) # Replace uses of /dev/null with temporary files. @@ -164,12 +198,16 @@ def executeShCmd(cmd, cfg, cwd, results): named_temp_files.append(f.name) args[i] = f.name - procs.append(subprocess.Popen(args, cwd=cwd, - stdin = stdin, - stdout = stdout, - stderr = stderr, - env = cfg.environment, - close_fds = kUseCloseFDs)) + try: + procs.append(subprocess.Popen(args, cwd=cmd_shenv.cwd, + executable = executable, + stdin = stdin, + stdout = stdout, + stderr = stderr, + env = cmd_shenv.env, + close_fds = kUseCloseFDs)) + except OSError as e: + raise InternalShellError(j, 'Could not create process due to {}'.format(e)) # Immediately close stdin for any process taking stdin from us. if stdin == subprocess.PIPE: @@ -211,6 +249,11 @@ def executeShCmd(cmd, cfg, cwd, results): f.seek(0, 0) procData[i] = (procData[i][0], f.read()) + def to_string(bytes): + if isinstance(bytes, str): + return bytes + return bytes.encode('utf-8') + exitCode = None for i,(out,err) in enumerate(procData): res = procs[i].wait() @@ -218,10 +261,22 @@ def executeShCmd(cmd, cfg, cwd, results): if res == -signal.SIGINT: raise KeyboardInterrupt + # Ensure the resulting output is always of string type. + try: + out = to_string(out.decode('utf-8')) + except: + out = str(out) + try: + err = to_string(err.decode('utf-8')) + except: + err = str(err) + results.append((cmd.commands[i], out, err, res)) if cmd.pipe_err: # Python treats the exit code as a signed char. - if res < 0: + if exitCode is None: + exitCode = res + elif res < 0: exitCode = min(exitCode, res) else: exitCode = max(exitCode, res) @@ -241,98 +296,29 @@ def executeShCmd(cmd, cfg, cwd, results): return exitCode def executeScriptInternal(test, litConfig, tmpBase, commands, cwd): - ln = ' &&\n'.join(commands) - try: - cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse() - except: - return (Test.FAIL, "shell parser error on: %r" % ln) - - results = [] - try: - exitCode = executeShCmd(cmd, test.config, cwd, results) - except InternalShellError,e: - out = '' - err = e.message - exitCode = 255 - - out = err = '' - for i,(cmd, cmd_out,cmd_err,res) in enumerate(results): - out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args)) - out += 'Command %d Result: %r\n' % (i, res) - out += 'Command %d Output:\n%s\n\n' % (i, cmd_out) - out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err) - - return out, err, exitCode - -def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd): - import TclUtil cmds = [] for ln in commands: - # Given the unfortunate way LLVM's test are written, the line gets - # backslash substitution done twice. - ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True) - - try: - tokens = list(TclUtil.TclLexer(ln).lex()) - except: - return (Test.FAIL, "Tcl lexer error on: %r" % ln) - - # Validate there are no control tokens. - for t in tokens: - if not isinstance(t, str): - return (Test.FAIL, - "Invalid test line: %r containing %r" % (ln, t)) - try: - cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline()) + cmds.append(ShUtil.ShParser(ln, litConfig.isWindows, + test.config.pipefail).parse()) except: - return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln) - - if litConfig.useValgrind: - for pipeline in cmds: - if pipeline.commands: - # Only valgrind the first command in each pipeline, to avoid - # valgrinding things like grep, not, and FileCheck. - cmd = pipeline.commands[0] - cmd.args = litConfig.valgrindArgs + cmd.args + return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln) cmd = cmds[0] for c in cmds[1:]: cmd = ShUtil.Seq(cmd, '&&', c) - # FIXME: This is lame, we shouldn't need bash. See PR5240. - bashPath = litConfig.getBashPath() - if litConfig.useTclAsSh and bashPath: - script = tmpBase + '.script' - - # Write script file - f = open(script,'w') - print >>f, 'set -o pipefail' - cmd.toShell(f, pipefail = True) - f.close() - - if 0: - print >>sys.stdout, cmd - print >>sys.stdout, open(script).read() - print >>sys.stdout - return '', '', 0 - - command = [litConfig.getBashPath(), script] - out,err,exitCode = executeCommand(command, cwd=cwd, - env=test.config.environment) - - return out,err,exitCode - else: - results = [] - try: - exitCode = executeShCmd(cmd, test.config, cwd, results) - except InternalShellError,e: - results.append((e.command, '', e.message + '\n', 255)) - exitCode = 255 + results = [] + try: + shenv = ShellEnvironment(cwd, test.config.environment) + exitCode = executeShCmd(cmd, shenv, results) + except InternalShellError: + e = sys.exc_info()[1] + exitCode = 127 + results.append((e.command, '', e.message, exitCode)) out = err = '' - - for i,(cmd, cmd_out, cmd_err, res) in enumerate(results): + for i,(cmd, cmd_out,cmd_err,res) in enumerate(results): out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args)) out += 'Command %d Result: %r\n' % (i, res) out += 'Command %d Output:\n%s\n\n' % (i, cmd_out) @@ -348,11 +334,16 @@ def executeScript(test, litConfig, tmpBase, commands, cwd): script += '.bat' # Write script file - f = open(script,'w') + mode = 'w' + if litConfig.isWindows and not isWin32CMDEXE: + mode += 'b' # Avoid CRLFs when writing bash scripts. + f = open(script, mode) if isWin32CMDEXE: f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) else: - f.write(' &&\n'.join(commands)) + if test.config.pipefail: + f.write('set -o pipefail;') + f.write('{ ' + '; } &&\n{ '.join(commands) + '; }') f.write('\n') f.close() @@ -368,30 +359,71 @@ def executeScript(test, litConfig, tmpBase, commands, cwd): # run on clang with no real loss. command = litConfig.valgrindArgs + command - return executeCommand(command, cwd=cwd, env=test.config.environment) + return lit.util.executeCommand(command, cwd=cwd, + env=test.config.environment) -def isExpectedFail(xfails, xtargets, target_triple): - # Check if any xfail matches this target. - for item in xfails: - if item == '*' or item in target_triple: - break - else: - return False +def parseIntegratedTestScriptCommands(source_path): + """ + parseIntegratedTestScriptCommands(source_path) -> commands + + Parse the commands in an integrated test script file into a list of + (line_number, command_type, line). + """ - # If so, see if it is expected to pass on this target. + # This code is carefully written to be dual compatible with Python 2.5+ and + # Python 3 without requiring input files to always have valid codings. The + # trick we use is to open the file in binary mode and use the regular + # expression library to find the commands, with it scanning strings in + # Python2 and bytes in Python3. # - # FIXME: Rename XTARGET to something that makes sense, like XPASS. - for item in xtargets: - if item == '*' or item in target_triple: - return False + # Once we find a match, we do require each script line to be decodable to + # UTF-8, so we convert the outputs to UTF-8 before returning. This way the + # remaining code can work with "strings" agnostic of the executing Python + # version. + + keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.'] + keywords_re = re.compile( + to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),))) + + f = open(source_path, 'rb') + try: + # Read the entire file contents. + data = f.read() + + # Ensure the data ends with a newline. + if not data.endswith(to_bytes('\n')): + data = data + to_bytes('\n') + + # Iterate over the matches. + line_number = 1 + last_match_position = 0 + for match in keywords_re.finditer(data): + # Compute the updated line number by counting the intervening + # newlines. + match_position = match.start() + line_number += data.count(to_bytes('\n'), last_match_position, + match_position) + last_match_position = match_position + + # Convert the keyword and line to UTF-8 strings and yield the + # command. Note that we take care to return regular strings in + # Python 2, to avoid other code having to differentiate between the + # str and unicode types. + keyword,ln = match.groups() + yield (line_number, to_string(keyword[:-1].decode('utf-8')), + to_string(ln.decode('utf-8'))) + finally: + f.close() - return True def parseIntegratedTestScript(test, normalize_slashes=False, - extra_substitutions=[]): + extra_substitutions=[], require_script=True): """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test - script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET' - information. The RUN lines also will have variable substitution performed. + script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' + and 'UNSUPPORTED' information. The RUN lines also will have variable + substitution performed. If 'require_script' is False an empty script may be + returned. This can be used for test formats where the actual script is + optional or ignored. """ # Get the temporary location, this is always relative to the test suite @@ -404,8 +436,6 @@ def parseIntegratedTestScript(test, normalize_slashes=False, execdir,execbase = os.path.split(execpath) tmpDir = os.path.join(execdir, 'Output') tmpBase = os.path.join(tmpDir, execbase) - if test.index is not None: - tmpBase += '_%d' % test.index # Normalize slashes, if requested. if normalize_slashes: @@ -424,42 +454,54 @@ def parseIntegratedTestScript(test, normalize_slashes=False, ('%{pathsep}', os.pathsep), ('%t', tmpBase + '.tmp'), ('%T', tmpDir), - # FIXME: Remove this once we kill DejaGNU. - ('%abs_tmp', tmpBase + '.tmp'), ('#_MARKER_#', '%')]) + # "%/[STpst]" should be normalized. + substitutions.extend([ + ('%/s', sourcepath.replace('\\', '/')), + ('%/S', sourcedir.replace('\\', '/')), + ('%/p', sourcedir.replace('\\', '/')), + ('%/t', tmpBase.replace('\\', '/') + '.tmp'), + ('%/T', tmpDir.replace('\\', '/')), + ]) + # Collect the test lines from the script. script = [] - xfails = [] - xtargets = [] requires = [] - for ln in open(sourcepath): - if 'RUN:' in ln: - # Isolate the command to run. - index = ln.index('RUN:') - ln = ln[index+4:] - + unsupported = [] + for line_number, command_type, ln in \ + parseIntegratedTestScriptCommands(sourcepath): + if command_type == 'RUN': # Trim trailing whitespace. ln = ln.rstrip() + # Substitute line number expressions + ln = re.sub('%\(line\)', str(line_number), ln) + def replace_line_number(match): + if match.group(1) == '+': + return str(line_number + int(match.group(2))) + if match.group(1) == '-': + return str(line_number - int(match.group(2))) + ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln) + # Collapse lines with trailing '\\'. if script and script[-1][-1] == '\\': script[-1] = script[-1][:-1] + ln else: script.append(ln) - elif 'XFAIL:' in ln: - items = ln[ln.index('XFAIL:') + 6:].split(',') - xfails.extend([s.strip() for s in items]) - elif 'XTARGET:' in ln: - items = ln[ln.index('XTARGET:') + 8:].split(',') - xtargets.extend([s.strip() for s in items]) - elif 'REQUIRES:' in ln: - items = ln[ln.index('REQUIRES:') + 9:].split(',') - requires.extend([s.strip() for s in items]) - elif 'END.' in ln: - # Check for END. lines. - if ln[ln.index('END.'):].strip() == 'END.': + elif command_type == 'XFAIL': + test.xfails.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'REQUIRES': + requires.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'UNSUPPORTED': + unsupported.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'END': + # END commands are only honored if the rest of the line is empty. + if not ln.strip(): break + else: + raise ValueError("unknown script command type: %r" % ( + command_type,)) # Apply substitutions to the script. Allow full regular # expression syntax. Replace each matching occurrence of regular @@ -473,96 +515,80 @@ def parseIntegratedTestScript(test, normalize_slashes=False, # Strip the trailing newline and any extra whitespace. return ln.strip() - script = map(processLine, script) + script = [processLine(ln) + for ln in script] # Verify the script contains a run line. - if not script: - return (Test.UNRESOLVED, "Test has no run line!") + if require_script and not script: + return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!") # Check for unterminated run lines. - if script[-1][-1] == '\\': - return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')") + if script and script[-1][-1] == '\\': + return lit.Test.Result(Test.UNRESOLVED, + "Test has unterminated run lines (with '\\')") # Check that we have the required features: missing_required_features = [f for f in requires if f not in test.config.available_features] if missing_required_features: msg = ', '.join(missing_required_features) - return (Test.UNSUPPORTED, - "Test requires the following features: %s" % msg) - - isXFail = isExpectedFail(xfails, xtargets, test.suite.config.target_triple) - return script,isXFail,tmpBase,execdir - -def formatTestOutput(status, out, err, exitCode, failDueToStderr, script): - output = StringIO.StringIO() - print >>output, "Script:" - print >>output, "--" - print >>output, '\n'.join(script) - print >>output, "--" - print >>output, "Exit Code: %r" % exitCode, - if failDueToStderr: - print >>output, "(but there was output on stderr)" - else: - print >>output - if out: - print >>output, "Command Output (stdout):" - print >>output, "--" - output.write(out) - print >>output, "--" - if err: - print >>output, "Command Output (stderr):" - print >>output, "--" - output.write(err) - print >>output, "--" - return (status, output.getvalue()) - -def executeTclTest(test, litConfig): - if test.config.unsupported: - return (Test.UNSUPPORTED, 'Test is unsupported') - - # Parse the test script, normalizing slashes in substitutions on Windows - # (since otherwise Tcl style lexing will treat them as escapes). - res = parseIntegratedTestScript(test, normalize_slashes=kIsWindows) - if len(res) == 2: - return res - - script, isXFail, tmpBase, execdir = res - - if litConfig.noExecute: - return (Test.PASS, '') - + return lit.Test.Result(Test.UNSUPPORTED, + "Test requires the following features: %s" % msg) + unsupported_features = [f for f in unsupported + if f in test.config.available_features] + if unsupported_features: + msg = ', '.join(unsupported_features) + return lit.Test.Result(Test.UNSUPPORTED, + "Test is unsupported with the following features: %s" % msg) + + unsupported_targets = [f for f in unsupported + if f in test.suite.config.target_triple] + if unsupported_targets: + return lit.Test.Result(Test.UNSUPPORTED, + "Test is unsupported with the following triple: %s" % ( + test.suite.config.target_triple,)) + + if test.config.limit_to_features: + # Check that we have one of the limit_to_features features in requires. + limit_to_features_tests = [f for f in test.config.limit_to_features + if f in requires] + if not limit_to_features_tests: + msg = ', '.join(test.config.limit_to_features) + return lit.Test.Result(Test.UNSUPPORTED, + "Test requires one of the limit_to_features features %s" % msg) + + return script,tmpBase,execdir + +def _runShTest(test, litConfig, useExternalSh, + script, tmpBase, execdir): # Create the output directory if it does not already exist. - Util.mkdir_p(os.path.dirname(tmpBase)) + lit.util.mkdir_p(os.path.dirname(tmpBase)) - res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir) - if len(res) == 2: + if useExternalSh: + res = executeScript(test, litConfig, tmpBase, script, execdir) + else: + res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) + if isinstance(res, lit.Test.Result): return res - # Test for failure. In addition to the exit code, Tcl commands are - # considered to fail if there is any standard error output. out,err,exitCode = res - if isXFail: - ok = exitCode != 0 or err and not litConfig.ignoreStdErr - if ok: - status = Test.XFAIL - else: - status = Test.XPASS + if exitCode == 0: + status = Test.PASS else: - ok = exitCode == 0 and (not err or litConfig.ignoreStdErr) - if ok: - status = Test.PASS - else: - status = Test.FAIL + status = Test.FAIL - if ok: - return (status,'') + # Form the output log. + output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % ( + '\n'.join(script), exitCode) - # Set a flag for formatTestOutput so it can explain why the test was - # considered to have failed, despite having an exit code of 0. - failDueToStderr = exitCode == 0 and err and not litConfig.ignoreStdErr + # Append the outputs, if present. + if out: + output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) + if err: + output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) + + return lit.Test.Result(status, output) - return formatTestOutput(status, out, err, exitCode, failDueToStderr, script) def executeShTest(test, litConfig, useExternalSh, extra_substitutions=[]): @@ -570,42 +596,23 @@ def executeShTest(test, litConfig, useExternalSh, return (Test.UNSUPPORTED, 'Test is unsupported') res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions) - if len(res) == 2: + if isinstance(res, lit.Test.Result): return res - - script, isXFail, tmpBase, execdir = res - if litConfig.noExecute: - return (Test.PASS, '') + return lit.Test.Result(Test.PASS) - # Create the output directory if it does not already exist. - Util.mkdir_p(os.path.dirname(tmpBase)) + script, tmpBase, execdir = res - if useExternalSh: - res = executeScript(test, litConfig, tmpBase, script, execdir) - else: - res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) - if len(res) == 2: - return res - - out,err,exitCode = res - if isXFail: - ok = exitCode != 0 - if ok: - status = Test.XFAIL - else: - status = Test.XPASS - else: - ok = exitCode == 0 - if ok: - status = Test.PASS - else: - status = Test.FAIL - - if ok: - return (status,'') - - # Sh tests are not considered to fail just from stderr output. - failDueToStderr = False - - return formatTestOutput(status, out, err, exitCode, failDueToStderr, script) + # Re-run failed tests up to test_retry_attempts times. + attempts = 1 + if hasattr(test.config, 'test_retry_attempts'): + attempts += test.config.test_retry_attempts + for i in range(attempts): + res = _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir) + if res.code != Test.FAIL: + break + # If we had to run the test more than once, count it as a flaky pass. These + # will be printed separately in the test summary. + if i > 0 and res.code == Test.PASS: + res.code = Test.FLAKYPASS + return res