X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;ds=sidebyside;f=utils%2Flit%2Flit%2FTestRunner.py;h=24075ff94048e4667001db56bb1dff03ee732948;hb=47f0e3f434e2e43f951c3a826c40906cb15b7285;hp=54bbd68893426737f9ada540ec18ac266233eabd;hpb=493e8d4bf503c8f573e0dce95ea146d181c623c6;p=oota-llvm.git diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py index 54bbd688934..24075ff9404 100644 --- a/utils/lit/lit/TestRunner.py +++ b/utils/lit/lit/TestRunner.py @@ -3,14 +3,11 @@ import os, signal, subprocess, sys import re import platform import tempfile -try: - from io import StringIO -except ImportError: - from StringIO import StringIO import lit.ShUtil as ShUtil import lit.Test as Test import lit.util +from lit.util import to_bytes, to_string class InternalShellError(Exception): def __init__(self, command, message): @@ -25,33 +22,60 @@ kUseCloseFDs = not kIsWindows # Use temporary files to replace /dev/null on Windows. kAvoidDevNull = kIsWindows -def executeShCmd(cmd, cfg, cwd, results): +class ShellEnvironment(object): + + """Mutable shell environment containing things like CWD and env vars. + + Environment variables are not implemented, but cwd tracking is. + """ + + def __init__(self, cwd, env): + self.cwd = cwd + self.env = dict(env) + +def executeShCmd(cmd, shenv, results): if isinstance(cmd, ShUtil.Seq): if cmd.op == ';': - res = executeShCmd(cmd.lhs, cfg, cwd, results) - return executeShCmd(cmd.rhs, cfg, cwd, results) + res = executeShCmd(cmd.lhs, shenv, results) + return executeShCmd(cmd.rhs, shenv, results) if cmd.op == '&': raise InternalShellError(cmd,"unsupported shell operator: '&'") if cmd.op == '||': - res = executeShCmd(cmd.lhs, cfg, cwd, results) + res = executeShCmd(cmd.lhs, shenv, results) if res != 0: - res = executeShCmd(cmd.rhs, cfg, cwd, results) + res = executeShCmd(cmd.rhs, shenv, results) return res if cmd.op == '&&': - res = executeShCmd(cmd.lhs, cfg, cwd, results) + res = executeShCmd(cmd.lhs, shenv, results) if res is None: return res if res == 0: - res = executeShCmd(cmd.rhs, cfg, cwd, results) + res = executeShCmd(cmd.rhs, shenv, results) return res raise ValueError('Unknown shell command: %r' % cmd.op) - assert isinstance(cmd, ShUtil.Pipeline) + + # Handle shell builtins first. + if cmd.commands[0].args[0] == 'cd': + if len(cmd.commands) != 1: + raise ValueError("'cd' cannot be part of a pipeline") + if len(cmd.commands[0].args) != 2: + raise ValueError("'cd' supports only one argument") + newdir = cmd.commands[0].args[1] + # Update the cwd in the parent environment. + if os.path.isabs(newdir): + shenv.cwd = newdir + else: + shenv.cwd = os.path.join(shenv.cwd, newdir) + # The cd builtin always succeeds. If the directory does not exist, the + # following Popen calls will fail instead. + return 0 + procs = [] input = subprocess.PIPE stderrTempFiles = [] @@ -61,6 +85,23 @@ def executeShCmd(cmd, cfg, cwd, results): # output. This is null until we have seen some output using # stderr. for i,j in enumerate(cmd.commands): + # Reference the global environment by default. + cmd_shenv = shenv + if j.args[0] == 'env': + # Create a copy of the global environment and modify it for this one + # command. There might be multiple envs in a pipeline: + # env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s + cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env) + arg_idx = 1 + for arg_idx, arg in enumerate(j.args[1:]): + # Partition the string into KEY=VALUE. + key, eq, val = arg.partition('=') + # Stop if there was no equals. + if eq == '': + break + cmd_shenv.env[key] = val + j.args = j.args[arg_idx+1:] + # Apply the redirections, we use (N,) as a sentinel to indicate stdin, # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or # from a file are represented with a list [file, mode, file-object] @@ -105,7 +146,9 @@ def executeShCmd(cmd, cfg, cwd, results): if kAvoidDevNull and r[0] == '/dev/null': r[2] = tempfile.TemporaryFile(mode=r[1]) else: - r[2] = open(r[0], r[1]) + # Make sure relative paths are relative to the cwd. + redir_filename = os.path.join(cmd_shenv.cwd, r[0]) + r[2] = open(redir_filename, r[1]) # Workaround a Win32 and/or subprocess bug when appending. # # FIXME: Actually, this is probably an instance of PR6753. @@ -135,8 +178,15 @@ def executeShCmd(cmd, cfg, cwd, results): # Resolve the executable path ourselves. args = list(j.args) - args[0] = lit.util.which(args[0], cfg.environment['PATH']) - if not args[0]: + executable = None + # For paths relative to cwd, use the cwd of the shell environment. + if args[0].startswith('.'): + exe_in_cwd = os.path.join(cmd_shenv.cwd, args[0]) + if os.path.isfile(exe_in_cwd): + executable = exe_in_cwd + if not executable: + executable = lit.util.which(args[0], cmd_shenv.env['PATH']) + if not executable: raise InternalShellError(j, '%r: command not found' % j.args[0]) # Replace uses of /dev/null with temporary files. @@ -148,12 +198,16 @@ def executeShCmd(cmd, cfg, cwd, results): named_temp_files.append(f.name) args[i] = f.name - procs.append(subprocess.Popen(args, cwd=cwd, - stdin = stdin, - stdout = stdout, - stderr = stderr, - env = cfg.environment, - close_fds = kUseCloseFDs)) + try: + procs.append(subprocess.Popen(args, cwd=cmd_shenv.cwd, + executable = executable, + stdin = stdin, + stdout = stdout, + stderr = stderr, + env = cmd_shenv.env, + close_fds = kUseCloseFDs)) + except OSError as e: + raise InternalShellError(j, 'Could not create process due to {}'.format(e)) # Immediately close stdin for any process taking stdin from us. if stdin == subprocess.PIPE: @@ -195,6 +249,11 @@ def executeShCmd(cmd, cfg, cwd, results): f.seek(0, 0) procData[i] = (procData[i][0], f.read()) + def to_string(bytes): + if isinstance(bytes, str): + return bytes + return bytes.encode('utf-8') + exitCode = None for i,(out,err) in enumerate(procData): res = procs[i].wait() @@ -202,6 +261,16 @@ def executeShCmd(cmd, cfg, cwd, results): if res == -signal.SIGINT: raise KeyboardInterrupt + # Ensure the resulting output is always of string type. + try: + out = to_string(out.decode('utf-8')) + except: + out = str(out) + try: + err = to_string(err.decode('utf-8')) + except: + err = str(err) + results.append((cmd.commands[i], out, err, res)) if cmd.pipe_err: # Python treats the exit code as a signed char. @@ -233,7 +302,7 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd): cmds.append(ShUtil.ShParser(ln, litConfig.isWindows, test.config.pipefail).parse()) except: - return (Test.FAIL, "shell parser error on: %r" % ln) + return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln) cmd = cmds[0] for c in cmds[1:]: @@ -241,7 +310,8 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd): results = [] try: - exitCode = executeShCmd(cmd, test.config, cwd, results) + shenv = ShellEnvironment(cwd, test.config.environment) + exitCode = executeShCmd(cmd, shenv, results) except InternalShellError: e = sys.exc_info()[1] exitCode = 127 @@ -292,28 +362,68 @@ def executeScript(test, litConfig, tmpBase, commands, cwd): return lit.util.executeCommand(command, cwd=cwd, env=test.config.environment) -def isExpectedFail(test, xfails): - # Check if any of the xfails match an available feature or the target. - for item in xfails: - # If this is the wildcard, it always fails. - if item == '*': - return True +def parseIntegratedTestScriptCommands(source_path): + """ + parseIntegratedTestScriptCommands(source_path) -> commands + + Parse the commands in an integrated test script file into a list of + (line_number, command_type, line). + """ + + # This code is carefully written to be dual compatible with Python 2.5+ and + # Python 3 without requiring input files to always have valid codings. The + # trick we use is to open the file in binary mode and use the regular + # expression library to find the commands, with it scanning strings in + # Python2 and bytes in Python3. + # + # Once we find a match, we do require each script line to be decodable to + # UTF-8, so we convert the outputs to UTF-8 before returning. This way the + # remaining code can work with "strings" agnostic of the executing Python + # version. - # If this is an exact match for one of the features, it fails. - if item in test.config.available_features: - return True + keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.'] + keywords_re = re.compile( + to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),))) - # If this is a part of the target triple, it fails. - if item in test.suite.config.target_triple: - return True + f = open(source_path, 'rb') + try: + # Read the entire file contents. + data = f.read() + + # Ensure the data ends with a newline. + if not data.endswith(to_bytes('\n')): + data = data + to_bytes('\n') + + # Iterate over the matches. + line_number = 1 + last_match_position = 0 + for match in keywords_re.finditer(data): + # Compute the updated line number by counting the intervening + # newlines. + match_position = match.start() + line_number += data.count(to_bytes('\n'), last_match_position, + match_position) + last_match_position = match_position + + # Convert the keyword and line to UTF-8 strings and yield the + # command. Note that we take care to return regular strings in + # Python 2, to avoid other code having to differentiate between the + # str and unicode types. + keyword,ln = match.groups() + yield (line_number, to_string(keyword[:-1].decode('utf-8')), + to_string(ln.decode('utf-8'))) + finally: + f.close() - return False def parseIntegratedTestScript(test, normalize_slashes=False, - extra_substitutions=[]): + extra_substitutions=[], require_script=True): """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' - information. The RUN lines also will have variable substitution performed. + and 'UNSUPPORTED' information. The RUN lines also will have variable + substitution performed. If 'require_script' is False an empty script may be + returned. This can be used for test formats where the actual script is + optional or ignored. """ # Get the temporary location, this is always relative to the test suite @@ -357,16 +467,11 @@ def parseIntegratedTestScript(test, normalize_slashes=False, # Collect the test lines from the script. script = [] - xfails = [] requires = [] - line_number = 0 - for ln in open(sourcepath): - line_number += 1 - if 'RUN:' in ln: - # Isolate the command to run. - index = ln.index('RUN:') - ln = ln[index+4:] - + unsupported = [] + for line_number, command_type, ln in \ + parseIntegratedTestScriptCommands(sourcepath): + if command_type == 'RUN': # Trim trailing whitespace. ln = ln.rstrip() @@ -384,16 +489,19 @@ def parseIntegratedTestScript(test, normalize_slashes=False, script[-1] = script[-1][:-1] + ln else: script.append(ln) - elif 'XFAIL:' in ln: - items = ln[ln.index('XFAIL:') + 6:].split(',') - xfails.extend([s.strip() for s in items]) - elif 'REQUIRES:' in ln: - items = ln[ln.index('REQUIRES:') + 9:].split(',') - requires.extend([s.strip() for s in items]) - elif 'END.' in ln: - # Check for END. lines. - if ln[ln.index('END.'):].strip() == 'END.': + elif command_type == 'XFAIL': + test.xfails.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'REQUIRES': + requires.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'UNSUPPORTED': + unsupported.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'END': + # END commands are only honored if the rest of the line is empty. + if not ln.strip(): break + else: + raise ValueError("unknown script command type: %r" % ( + command_type,)) # Apply substitutions to the script. Allow full regular # expression syntax. Replace each matching occurrence of regular @@ -411,57 +519,48 @@ def parseIntegratedTestScript(test, normalize_slashes=False, for ln in script] # Verify the script contains a run line. - if not script: - return (Test.UNRESOLVED, "Test has no run line!") + if require_script and not script: + return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!") # Check for unterminated run lines. - if script[-1][-1] == '\\': - return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')") + if script and script[-1][-1] == '\\': + return lit.Test.Result(Test.UNRESOLVED, + "Test has unterminated run lines (with '\\')") # Check that we have the required features: missing_required_features = [f for f in requires if f not in test.config.available_features] if missing_required_features: msg = ', '.join(missing_required_features) - return (Test.UNSUPPORTED, - "Test requires the following features: %s" % msg) - - isXFail = isExpectedFail(test, xfails) - return script,isXFail,tmpBase,execdir - -def formatTestOutput(status, out, err, exitCode, script): - output = StringIO() - output.write(u"Script:\n") - output.write(u"--\n") - output.write(u'\n'.join(script)) - output.write(u"\n--\n") - output.write(u"Exit Code: %r\n\n" % exitCode) - if out: - output.write(u"Command Output (stdout):\n") - output.write(u"--\n") - output.write(unicode(out)) - output.write(u"--\n") - if err: - output.write(u"Command Output (stderr):\n") - output.write(u"--\n") - output.write(unicode(err)) - output.write(u"--\n") - return (status, output.getvalue()) - -def executeShTest(test, litConfig, useExternalSh, - extra_substitutions=[]): - if test.config.unsupported: - return (Test.UNSUPPORTED, 'Test is unsupported') - - res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions) - if len(res) == 2: - return res - - script, isXFail, tmpBase, execdir = res - - if litConfig.noExecute: - return (Test.PASS, '') - + return lit.Test.Result(Test.UNSUPPORTED, + "Test requires the following features: %s" % msg) + unsupported_features = [f for f in unsupported + if f in test.config.available_features] + if unsupported_features: + msg = ', '.join(unsupported_features) + return lit.Test.Result(Test.UNSUPPORTED, + "Test is unsupported with the following features: %s" % msg) + + unsupported_targets = [f for f in unsupported + if f in test.suite.config.target_triple] + if unsupported_targets: + return lit.Test.Result(Test.UNSUPPORTED, + "Test is unsupported with the following triple: %s" % ( + test.suite.config.target_triple,)) + + if test.config.limit_to_features: + # Check that we have one of the limit_to_features features in requires. + limit_to_features_tests = [f for f in test.config.limit_to_features + if f in requires] + if not limit_to_features_tests: + msg = ', '.join(test.config.limit_to_features) + return lit.Test.Result(Test.UNSUPPORTED, + "Test requires one of the limit_to_features features %s" % msg) + + return script,tmpBase,execdir + +def _runShTest(test, litConfig, useExternalSh, + script, tmpBase, execdir): # Create the output directory if it does not already exist. lit.util.mkdir_p(os.path.dirname(tmpBase)) @@ -469,24 +568,51 @@ def executeShTest(test, litConfig, useExternalSh, res = executeScript(test, litConfig, tmpBase, script, execdir) else: res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) - if len(res) == 2: + if isinstance(res, lit.Test.Result): return res out,err,exitCode = res - if isXFail: - ok = exitCode != 0 - if ok: - status = Test.XFAIL - else: - status = Test.XPASS + if exitCode == 0: + status = Test.PASS else: - ok = exitCode == 0 - if ok: - status = Test.PASS - else: - status = Test.FAIL + status = Test.FAIL - if ok: - return (status,'') + # Form the output log. + output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % ( + '\n'.join(script), exitCode) - return formatTestOutput(status, out, err, exitCode, script) + # Append the outputs, if present. + if out: + output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) + if err: + output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) + + return lit.Test.Result(status, output) + + +def executeShTest(test, litConfig, useExternalSh, + extra_substitutions=[]): + if test.config.unsupported: + return (Test.UNSUPPORTED, 'Test is unsupported') + + res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions) + if isinstance(res, lit.Test.Result): + return res + if litConfig.noExecute: + return lit.Test.Result(Test.PASS) + + script, tmpBase, execdir = res + + # Re-run failed tests up to test_retry_attempts times. + attempts = 1 + if hasattr(test.config, 'test_retry_attempts'): + attempts += test.config.test_retry_attempts + for i in range(attempts): + res = _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir) + if res.code != Test.FAIL: + break + # If we had to run the test more than once, count it as a flaky pass. These + # will be printed separately in the test summary. + if i > 0 and res.code == Test.PASS: + res.code = Test.FLAKYPASS + return res