1 from __future__ import absolute_import
2 import os, signal, subprocess, sys
8 import lit.ShUtil as ShUtil
9 import lit.Test as Test
11 from lit.util import to_bytes, to_string
13 class InternalShellError(Exception):
14 def __init__(self, command, message):
15 self.command = command
16 self.message = message
18 kIsWindows = platform.system() == 'Windows'
20 # Don't use close_fds on Windows.
21 kUseCloseFDs = not kIsWindows
23 # Use temporary files to replace /dev/null on Windows.
24 kAvoidDevNull = kIsWindows
26 class ShellEnvironment(object):
28 """Mutable shell environment containing things like CWD and env vars.
30 Environment variables are not implemented, but cwd tracking is.
33 def __init__(self, cwd, env):
37 class TimeoutHelper(object):
39 Object used to helper manage enforcing a timeout in
40 _executeShCmd(). It is passed through recursive calls
41 to collect processes that have been executed so that when
42 the timeout happens they can be killed.
44 def __init__(self, timeout):
45 self.timeout = timeout
47 self._timeoutReached = False
48 self._doneKillPass = False
49 # This lock will be used to protect concurrent access
50 # to _procs and _doneKillPass
60 return self.timeout > 0
62 def addProcess(self, proc):
67 self._procs.append(proc)
68 # Avoid re-entering the lock by finding out if kill needs to be run
69 # again here but call it if necessary once we have left the lock.
70 # We could use a reentrant lock here instead but this code seems
72 needToRunKill = self._doneKillPass
74 # The initial call to _kill() from the timer thread already happened so
75 # we need to call it again from this thread, otherwise this process
76 # will be left to run even though the timeout was already hit
78 assert self.timeoutReached()
85 # Do some late initialisation that's only needed
86 # if there is a timeout set
87 self._lock = threading.Lock()
88 self._timer = threading.Timer(self.timeout, self._handleTimeoutReached)
91 def _handleTimeoutReached(self):
92 self._timeoutReached = True
95 def timeoutReached(self):
96 return self._timeoutReached
100 This method may be called multiple times as we might get unlucky
101 and be in the middle of creating a new process in _executeShCmd()
102 which won't yet be in ``self._procs``. By locking here and in
103 addProcess() we should be able to kill processes launched after
104 the initial call to _kill()
107 for p in self._procs:
108 lit.util.killProcessAndChildren(p.pid)
109 # Empty the list and note that we've done a pass over the list
110 self._procs = [] # Python2 doesn't have list.clear()
111 self._doneKillPass = True
113 def executeShCmd(cmd, shenv, results, timeout=0):
115 Wrapper around _executeShCmd that handles
118 # Use the helper even when no timeout is required to make
119 # other code simpler (i.e. avoid bunch of ``!= None`` checks)
120 timeoutHelper = TimeoutHelper(timeout)
122 timeoutHelper.startTimer()
123 finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper)
124 timeoutHelper.cancel()
126 if timeoutHelper.timeoutReached():
127 timeoutInfo = 'Reached timeout of {} seconds'.format(timeout)
129 return (finalExitCode, timeoutInfo)
131 def _executeShCmd(cmd, shenv, results, timeoutHelper):
132 if timeoutHelper.timeoutReached():
133 # Prevent further recursion if the timeout has been hit
134 # as we should try avoid launching more processes.
137 if isinstance(cmd, ShUtil.Seq):
139 res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper)
140 return _executeShCmd(cmd.rhs, shenv, results, timeoutHelper)
143 raise InternalShellError(cmd,"unsupported shell operator: '&'")
146 res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper)
148 res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper)
152 res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper)
157 res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper)
160 raise ValueError('Unknown shell command: %r' % cmd.op)
161 assert isinstance(cmd, ShUtil.Pipeline)
163 # Handle shell builtins first.
164 if cmd.commands[0].args[0] == 'cd':
165 if len(cmd.commands) != 1:
166 raise ValueError("'cd' cannot be part of a pipeline")
167 if len(cmd.commands[0].args) != 2:
168 raise ValueError("'cd' supports only one argument")
169 newdir = cmd.commands[0].args[1]
170 # Update the cwd in the parent environment.
171 if os.path.isabs(newdir):
174 shenv.cwd = os.path.join(shenv.cwd, newdir)
175 # The cd builtin always succeeds. If the directory does not exist, the
176 # following Popen calls will fail instead.
180 input = subprocess.PIPE
183 named_temp_files = []
184 # To avoid deadlock, we use a single stderr stream for piped
185 # output. This is null until we have seen some output using
187 for i,j in enumerate(cmd.commands):
188 # Reference the global environment by default.
190 if j.args[0] == 'env':
191 # Create a copy of the global environment and modify it for this one
192 # command. There might be multiple envs in a pipeline:
193 # env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s
194 cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env)
196 for arg_idx, arg in enumerate(j.args[1:]):
197 # Partition the string into KEY=VALUE.
198 key, eq, val = arg.partition('=')
199 # Stop if there was no equals.
202 cmd_shenv.env[key] = val
203 j.args = j.args[arg_idx+1:]
205 # Apply the redirections, we use (N,) as a sentinel to indicate stdin,
206 # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
207 # from a file are represented with a list [file, mode, file-object]
208 # where file-object is initially None.
209 redirects = [(0,), (1,), (2,)]
210 for r in j.redirects:
212 redirects[2] = [r[1], 'w', None]
213 elif r[0] == ('>>',2):
214 redirects[2] = [r[1], 'a', None]
215 elif r[0] == ('>&',2) and r[1] in '012':
216 redirects[2] = redirects[int(r[1])]
217 elif r[0] == ('>&',) or r[0] == ('&>',):
218 redirects[1] = redirects[2] = [r[1], 'w', None]
220 redirects[1] = [r[1], 'w', None]
221 elif r[0] == ('>>',):
222 redirects[1] = [r[1], 'a', None]
224 redirects[0] = [r[1], 'r', None]
226 raise InternalShellError(j,"Unsupported redirect: %r" % (r,))
228 # Map from the final redirections to something subprocess can handle.
230 for index,r in enumerate(redirects):
235 raise InternalShellError(j,"Unsupported redirect for stdin")
237 result = subprocess.PIPE
239 result = subprocess.STDOUT
242 raise InternalShellError(j,"Unsupported redirect on stdout")
243 result = subprocess.PIPE
246 if kAvoidDevNull and r[0] == '/dev/null':
247 r[2] = tempfile.TemporaryFile(mode=r[1])
249 # Make sure relative paths are relative to the cwd.
250 redir_filename = os.path.join(cmd_shenv.cwd, r[0])
251 r[2] = open(redir_filename, r[1])
252 # Workaround a Win32 and/or subprocess bug when appending.
254 # FIXME: Actually, this is probably an instance of PR6753.
257 opened_files.append(r[2])
259 final_redirects.append(result)
261 stdin, stdout, stderr = final_redirects
263 # If stderr wants to come from stdout, but stdout isn't a pipe, then put
264 # stderr on a pipe and treat it as stdout.
265 if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
266 stderr = subprocess.PIPE
267 stderrIsStdout = True
269 stderrIsStdout = False
271 # Don't allow stderr on a PIPE except for the last
272 # process, this could deadlock.
274 # FIXME: This is slow, but so is deadlock.
275 if stderr == subprocess.PIPE and j != cmd.commands[-1]:
276 stderr = tempfile.TemporaryFile(mode='w+b')
277 stderrTempFiles.append((i, stderr))
279 # Resolve the executable path ourselves.
282 # For paths relative to cwd, use the cwd of the shell environment.
283 if args[0].startswith('.'):
284 exe_in_cwd = os.path.join(cmd_shenv.cwd, args[0])
285 if os.path.isfile(exe_in_cwd):
286 executable = exe_in_cwd
288 executable = lit.util.which(args[0], cmd_shenv.env['PATH'])
290 raise InternalShellError(j, '%r: command not found' % j.args[0])
292 # Replace uses of /dev/null with temporary files.
294 for i,arg in enumerate(args):
295 if arg == "/dev/null":
296 f = tempfile.NamedTemporaryFile(delete=False)
298 named_temp_files.append(f.name)
302 procs.append(subprocess.Popen(args, cwd=cmd_shenv.cwd,
303 executable = executable,
308 close_fds = kUseCloseFDs))
309 # Let the helper know about this process
310 timeoutHelper.addProcess(procs[-1])
312 raise InternalShellError(j, 'Could not create process ({}) due to {}'.format(executable, e))
314 # Immediately close stdin for any process taking stdin from us.
315 if stdin == subprocess.PIPE:
316 procs[-1].stdin.close()
317 procs[-1].stdin = None
319 # Update the current stdin source.
320 if stdout == subprocess.PIPE:
321 input = procs[-1].stdout
323 input = procs[-1].stderr
325 input = subprocess.PIPE
327 # Explicitly close any redirected files. We need to do this now because we
328 # need to release any handles we may have on the temporary files (important
329 # on Win32, for example). Since we have already spawned the subprocess, our
330 # handles have already been transferred so we do not need them anymore.
331 for f in opened_files:
334 # FIXME: There is probably still deadlock potential here. Yawn.
335 procData = [None] * len(procs)
336 procData[-1] = procs[-1].communicate()
338 for i in range(len(procs) - 1):
339 if procs[i].stdout is not None:
340 out = procs[i].stdout.read()
343 if procs[i].stderr is not None:
344 err = procs[i].stderr.read()
347 procData[i] = (out,err)
349 # Read stderr out of the temp files.
350 for i,f in stderrTempFiles:
352 procData[i] = (procData[i][0], f.read())
354 def to_string(bytes):
355 if isinstance(bytes, str):
357 return bytes.encode('utf-8')
360 for i,(out,err) in enumerate(procData):
361 res = procs[i].wait()
362 # Detect Ctrl-C in subprocess.
363 if res == -signal.SIGINT:
364 raise KeyboardInterrupt
366 # Ensure the resulting output is always of string type.
368 out = to_string(out.decode('utf-8'))
372 err = to_string(err.decode('utf-8'))
376 results.append((cmd.commands[i], out, err, res, timeoutHelper.timeoutReached()))
378 # Python treats the exit code as a signed char.
382 exitCode = min(exitCode, res)
384 exitCode = max(exitCode, res)
388 # Remove any named temporary files we created.
389 for f in named_temp_files:
396 exitCode = not exitCode
400 def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
404 cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
405 test.config.pipefail).parse())
407 return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln)
411 cmd = ShUtil.Seq(cmd, '&&', c)
416 shenv = ShellEnvironment(cwd, test.config.environment)
417 exitCode, timeoutInfo = executeShCmd(cmd, shenv, results, timeout=litConfig.maxIndividualTestTime)
418 except InternalShellError:
419 e = sys.exc_info()[1]
421 results.append((e.command, '', e.message, exitCode, False))
424 for i,(cmd, cmd_out, cmd_err, res, timeoutReached) in enumerate(results):
425 out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
426 out += 'Command %d Result: %r\n' % (i, res)
427 if litConfig.maxIndividualTestTime > 0:
428 out += 'Command %d Reached Timeout: %s\n\n' % (i, str(timeoutReached))
429 out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
430 out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
432 return out, err, exitCode, timeoutInfo
434 def executeScript(test, litConfig, tmpBase, commands, cwd):
435 bashPath = litConfig.getBashPath();
436 isWin32CMDEXE = (litConfig.isWindows and not bashPath)
437 script = tmpBase + '.script'
443 if litConfig.isWindows and not isWin32CMDEXE:
444 mode += 'b' # Avoid CRLFs when writing bash scripts.
445 f = open(script, mode)
447 f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
449 if test.config.pipefail:
450 f.write('set -o pipefail;')
451 f.write('{ ' + '; } &&\n{ '.join(commands) + '; }')
456 command = ['cmd','/c', script]
459 command = [bashPath, script]
461 command = ['/bin/sh', script]
462 if litConfig.useValgrind:
463 # FIXME: Running valgrind on sh is overkill. We probably could just
464 # run on clang with no real loss.
465 command = litConfig.valgrindArgs + command
468 out, err, exitCode = lit.util.executeCommand(command, cwd=cwd,
469 env=test.config.environment,
470 timeout=litConfig.maxIndividualTestTime)
471 return (out, err, exitCode, None)
472 except lit.util.ExecuteCommandTimeoutException as e:
473 return (e.out, e.err, e.exitCode, e.msg)
475 def parseIntegratedTestScriptCommands(source_path, keywords):
477 parseIntegratedTestScriptCommands(source_path) -> commands
479 Parse the commands in an integrated test script file into a list of
480 (line_number, command_type, line).
483 # This code is carefully written to be dual compatible with Python 2.5+ and
484 # Python 3 without requiring input files to always have valid codings. The
485 # trick we use is to open the file in binary mode and use the regular
486 # expression library to find the commands, with it scanning strings in
487 # Python2 and bytes in Python3.
489 # Once we find a match, we do require each script line to be decodable to
490 # UTF-8, so we convert the outputs to UTF-8 before returning. This way the
491 # remaining code can work with "strings" agnostic of the executing Python
494 keywords_re = re.compile(
495 to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
497 f = open(source_path, 'rb')
499 # Read the entire file contents.
502 # Ensure the data ends with a newline.
503 if not data.endswith(to_bytes('\n')):
504 data = data + to_bytes('\n')
506 # Iterate over the matches.
508 last_match_position = 0
509 for match in keywords_re.finditer(data):
510 # Compute the updated line number by counting the intervening
512 match_position = match.start()
513 line_number += data.count(to_bytes('\n'), last_match_position,
515 last_match_position = match_position
517 # Convert the keyword and line to UTF-8 strings and yield the
518 # command. Note that we take care to return regular strings in
519 # Python 2, to avoid other code having to differentiate between the
520 # str and unicode types.
521 keyword,ln = match.groups()
522 yield (line_number, to_string(keyword[:-1].decode('utf-8')),
523 to_string(ln.decode('utf-8')))
527 def getTempPaths(test):
528 """Get the temporary location, this is always relative to the test suite
529 root, not test source root."""
530 execpath = test.getExecPath()
531 execdir,execbase = os.path.split(execpath)
532 tmpDir = os.path.join(execdir, 'Output')
533 tmpBase = os.path.join(tmpDir, execbase)
534 return tmpDir, tmpBase
536 def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
537 sourcepath = test.getSourcePath()
538 sourcedir = os.path.dirname(sourcepath)
540 # Normalize slashes, if requested.
541 if normalize_slashes:
542 sourcepath = sourcepath.replace('\\', '/')
543 sourcedir = sourcedir.replace('\\', '/')
544 tmpDir = tmpDir.replace('\\', '/')
545 tmpBase = tmpBase.replace('\\', '/')
547 # We use #_MARKER_# to hide %% while we do the other substitutions.
549 substitutions.extend([('%%', '#_MARKER_#')])
550 substitutions.extend(test.config.substitutions)
551 substitutions.extend([('%s', sourcepath),
554 ('%{pathsep}', os.pathsep),
555 ('%t', tmpBase + '.tmp'),
557 ('#_MARKER_#', '%')])
559 # "%/[STpst]" should be normalized.
560 substitutions.extend([
561 ('%/s', sourcepath.replace('\\', '/')),
562 ('%/S', sourcedir.replace('\\', '/')),
563 ('%/p', sourcedir.replace('\\', '/')),
564 ('%/t', tmpBase.replace('\\', '/') + '.tmp'),
565 ('%/T', tmpDir.replace('\\', '/')),
569 def applySubstitutions(script, substitutions):
570 """Apply substitutions to the script. Allow full regular expression syntax.
571 Replace each matching occurrence of regular expression pattern a with
572 substitution b in line ln."""
574 # Apply substitutions
575 for a,b in substitutions:
577 b = b.replace("\\","\\\\")
578 ln = re.sub(a, b, ln)
580 # Strip the trailing newline and any extra whitespace.
582 # Note Python 3 map() gives an iterator rather than a list so explicitly
583 # convert to list before returning.
584 return list(map(processLine, script))
586 def parseIntegratedTestScript(test, require_script=True):
587 """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
588 script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
589 and 'UNSUPPORTED' information. If 'require_script' is False an empty script
590 may be returned. This can be used for test formats where the actual script
591 is optional or ignored.
593 # Collect the test lines from the script.
594 sourcepath = test.getSourcePath()
598 keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.']
599 for line_number, command_type, ln in \
600 parseIntegratedTestScriptCommands(sourcepath, keywords):
601 if command_type == 'RUN':
602 # Trim trailing whitespace.
605 # Substitute line number expressions
606 ln = re.sub('%\(line\)', str(line_number), ln)
607 def replace_line_number(match):
608 if match.group(1) == '+':
609 return str(line_number + int(match.group(2)))
610 if match.group(1) == '-':
611 return str(line_number - int(match.group(2)))
612 ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln)
614 # Collapse lines with trailing '\\'.
615 if script and script[-1][-1] == '\\':
616 script[-1] = script[-1][:-1] + ln
619 elif command_type == 'XFAIL':
620 test.xfails.extend([s.strip() for s in ln.split(',')])
621 elif command_type == 'REQUIRES':
622 requires.extend([s.strip() for s in ln.split(',')])
623 elif command_type == 'UNSUPPORTED':
624 unsupported.extend([s.strip() for s in ln.split(',')])
625 elif command_type == 'END':
626 # END commands are only honored if the rest of the line is empty.
630 raise ValueError("unknown script command type: %r" % (
633 # Verify the script contains a run line.
634 if require_script and not script:
635 return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!")
637 # Check for unterminated run lines.
638 if script and script[-1][-1] == '\\':
639 return lit.Test.Result(Test.UNRESOLVED,
640 "Test has unterminated run lines (with '\\')")
642 # Check that we have the required features:
643 missing_required_features = [f for f in requires
644 if f not in test.config.available_features]
645 if missing_required_features:
646 msg = ', '.join(missing_required_features)
647 return lit.Test.Result(Test.UNSUPPORTED,
648 "Test requires the following features: %s" % msg)
649 unsupported_features = [f for f in unsupported
650 if f in test.config.available_features]
651 if unsupported_features:
652 msg = ', '.join(unsupported_features)
653 return lit.Test.Result(Test.UNSUPPORTED,
654 "Test is unsupported with the following features: %s" % msg)
656 unsupported_targets = [f for f in unsupported
657 if f in test.suite.config.target_triple]
658 if unsupported_targets:
659 return lit.Test.Result(Test.UNSUPPORTED,
660 "Test is unsupported with the following triple: %s" % (
661 test.suite.config.target_triple,))
663 if test.config.limit_to_features:
664 # Check that we have one of the limit_to_features features in requires.
665 limit_to_features_tests = [f for f in test.config.limit_to_features
667 if not limit_to_features_tests:
668 msg = ', '.join(test.config.limit_to_features)
669 return lit.Test.Result(Test.UNSUPPORTED,
670 "Test requires one of the limit_to_features features %s" % msg)
674 def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
675 # Create the output directory if it does not already exist.
676 lit.util.mkdir_p(os.path.dirname(tmpBase))
678 execdir = os.path.dirname(test.getExecPath())
680 res = executeScript(test, litConfig, tmpBase, script, execdir)
682 res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
683 if isinstance(res, lit.Test.Result):
686 out,err,exitCode,timeoutInfo = res
690 if timeoutInfo == None:
693 status = Test.TIMEOUT
695 # Form the output log.
696 output = """Script:\n--\n%s\n--\nExit Code: %d\n""" % (
697 '\n'.join(script), exitCode)
699 if timeoutInfo != None:
700 output += """Timeout: %s\n""" % (timeoutInfo,)
703 # Append the outputs, if present.
705 output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
707 output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
709 return lit.Test.Result(status, output)
712 def executeShTest(test, litConfig, useExternalSh,
713 extra_substitutions=[]):
714 if test.config.unsupported:
715 return (Test.UNSUPPORTED, 'Test is unsupported')
717 script = parseIntegratedTestScript(test)
718 if isinstance(script, lit.Test.Result):
720 if litConfig.noExecute:
721 return lit.Test.Result(Test.PASS)
723 tmpDir, tmpBase = getTempPaths(test)
724 substitutions = list(extra_substitutions)
725 substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
726 normalize_slashes=useExternalSh)
727 script = applySubstitutions(script, substitutions)
729 # Re-run failed tests up to test_retry_attempts times.
731 if hasattr(test.config, 'test_retry_attempts'):
732 attempts += test.config.test_retry_attempts
733 for i in range(attempts):
734 res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
735 if res.code != Test.FAIL:
737 # If we had to run the test more than once, count it as a flaky pass. These
738 # will be printed separately in the test summary.
739 if i > 0 and res.code == Test.PASS:
740 res.code = Test.FLAKYPASS