4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 import math, os, platform, random, re, sys, time, threading, traceback
15 from TestingConfig import TestingConfig
19 # Configuration files to look for when discovering test suites. These can be
20 # overridden with --config-prefix.
22 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
23 gConfigName = 'lit.cfg'
24 gSiteConfigName = 'lit.site.cfg'
26 kLocalConfigName = 'lit.local.cfg'
28 class TestingProgressDisplay:
29 def __init__(self, opts, numTests, progressBar=None):
31 self.numTests = numTests
33 self.lock = threading.Lock()
34 self.progressBar = progressBar
37 def update(self, test):
38 # Avoid locking overhead in quiet mode
39 if self.opts.quiet and not test.result.isFailure:
46 self.handleUpdate(test)
52 self.progressBar.clear()
55 elif self.opts.succinct:
56 sys.stdout.write('\n')
58 def handleUpdate(self, test):
61 self.progressBar.update(float(self.completed)/self.numTests,
64 if self.opts.succinct and not test.result.isFailure:
68 self.progressBar.clear()
70 print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
71 self.completed, self.numTests)
73 if test.result.isFailure and self.opts.showOutput:
74 print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
82 def __init__(self, tests, maxTime):
83 self.maxTime = maxTime
84 self.iter = iter(tests)
85 self.lock = threading.Lock()
86 self.startTime = time.time()
89 # Check if we have run out of time.
90 if self.maxTime is not None:
91 if time.time() - self.startTime > self.maxTime:
94 # Otherwise take the next test.
97 item = self.iter.next()
103 class Tester(threading.Thread):
104 def __init__(self, litConfig, provider, display):
105 threading.Thread.__init__(self)
106 self.litConfig = litConfig
107 self.provider = provider
108 self.display = display
112 item = self.provider.get()
117 def runTest(self, test):
119 startTime = time.time()
121 result, output = test.config.test_format.execute(test,
123 except KeyboardInterrupt:
124 # This is a sad hack. Unfortunately subprocess goes
125 # bonkers with ctrl-c and we start forking merrily.
126 print '\nCtrl-C detected, goodbye.'
129 if self.litConfig.debug:
131 result = Test.UNRESOLVED
132 output = 'Exception during script execution:\n'
133 output += traceback.format_exc()
135 elapsed = time.time() - startTime
137 test.setResult(result, output, elapsed)
138 self.display.update(test)
140 def dirContainsTestSuite(path):
141 cfgpath = os.path.join(path, gSiteConfigName)
142 if os.path.exists(cfgpath):
144 cfgpath = os.path.join(path, gConfigName)
145 if os.path.exists(cfgpath):
148 def getTestSuite(item, litConfig, cache):
149 """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
151 Find the test suite containing @arg item.
153 @retval (None, ...) - Indicates no test suite contains @arg item.
154 @retval (suite, relative_path) - The suite that @arg item is in, and its
155 relative path inside that suite.
158 # Check for a site config or a lit config.
159 cfgpath = dirContainsTestSuite(path)
161 # If we didn't find a config file, keep looking.
163 parent,base = os.path.split(path)
167 ts, relative = search(parent)
168 return (ts, relative + (base,))
170 # We found a config file, load it.
172 litConfig.note('loading suite config %r' % cfgpath)
174 cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
175 source_root = os.path.realpath(cfg.test_source_root or path)
176 exec_root = os.path.realpath(cfg.test_exec_root or path)
177 return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
180 # Check for an already instantiated test suite.
181 res = cache.get(path)
183 cache[path] = res = search1(path)
186 # Canonicalize the path.
187 item = os.path.realpath(item)
189 # Skip files and virtual components.
191 while not os.path.isdir(item):
192 parent,base = os.path.split(item)
195 components.append(base)
199 ts, relative = search(item)
200 return ts, tuple(relative + tuple(components))
202 def getLocalConfig(ts, path_in_suite, litConfig, cache):
203 def search1(path_in_suite):
204 # Get the parent config.
205 if not path_in_suite:
208 parent = search(path_in_suite[:-1])
210 # Load the local configuration.
211 source_path = ts.getSourcePath(path_in_suite)
212 cfgpath = os.path.join(source_path, kLocalConfigName)
214 litConfig.note('loading local config %r' % cfgpath)
215 return TestingConfig.frompath(cfgpath, parent, litConfig,
217 config = parent.clone(cfgpath))
219 def search(path_in_suite):
220 key = (ts, path_in_suite)
223 cache[key] = res = search1(path_in_suite)
226 return search(path_in_suite)
228 def getTests(path, litConfig, testSuiteCache, localConfigCache):
229 # Find the test suite for this input and its relative path.
230 ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
232 litConfig.warning('unable to find test suite for %r' % path)
236 litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
239 return getTestsInSuite(ts, path_in_suite, litConfig,
240 testSuiteCache, localConfigCache)
242 def getTestsInSuite(ts, path_in_suite, litConfig,
243 testSuiteCache, localConfigCache):
244 # Check that the source path exists (errors here are reported by the
246 source_path = ts.getSourcePath(path_in_suite)
247 if not os.path.exists(source_path):
250 # Check if the user named a test directly.
251 if not os.path.isdir(source_path):
252 lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
253 yield Test.Test(ts, path_in_suite, lc)
256 # Otherwise we have a directory to search for tests, start by getting the
257 # local configuration.
258 lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
261 for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
265 # Search subdirectories.
266 for filename in os.listdir(source_path):
267 # FIXME: This doesn't belong here?
268 if filename in ('Output', '.svn') or filename in lc.excludes:
271 # Ignore non-directories.
272 file_sourcepath = os.path.join(source_path, filename)
273 if not os.path.isdir(file_sourcepath):
276 # Check for nested test suites, first in the execpath in case there is a
277 # site configuration and then in the source path.
278 file_execpath = ts.getExecPath(path_in_suite + (filename,))
279 if dirContainsTestSuite(file_execpath):
280 subiter = getTests(file_execpath, litConfig,
281 testSuiteCache, localConfigCache)
282 elif dirContainsTestSuite(file_sourcepath):
283 subiter = getTests(file_sourcepath, litConfig,
284 testSuiteCache, localConfigCache)
286 # Otherwise, continue loading from inside this test suite.
287 subiter = getTestsInSuite(ts, path_in_suite + (filename,),
288 litConfig, testSuiteCache,
294 def runTests(numThreads, litConfig, provider, display):
295 # If only using one testing thread, don't use threads at all; this lets us
296 # profile, among other things.
298 t = Tester(litConfig, provider, display)
302 # Otherwise spin up the testing threads and wait for them to finish.
303 testers = [Tester(litConfig, provider, display)
304 for i in range(numThreads)]
310 except KeyboardInterrupt:
315 from optparse import OptionParser, OptionGroup
316 parser = OptionParser("usage: %prog [options] {file-or-path}")
318 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
319 help="Number of testing threads",
320 type=int, action="store", default=None)
321 parser.add_option("", "--config-prefix", dest="configPrefix",
322 metavar="NAME", help="Prefix for 'lit' config files",
323 action="store", default=None)
324 parser.add_option("", "--param", dest="userParameters",
326 help="Add 'NAME' = 'VAL' to the user defined parameters",
327 type=str, action="append", default=[])
329 group = OptionGroup(parser, "Output Format")
330 # FIXME: I find these names very confusing, although I like the
332 group.add_option("-q", "--quiet", dest="quiet",
333 help="Suppress no error output",
334 action="store_true", default=False)
335 group.add_option("-s", "--succinct", dest="succinct",
336 help="Reduce amount of output",
337 action="store_true", default=False)
338 group.add_option("-v", "--verbose", dest="showOutput",
339 help="Show all test output",
340 action="store_true", default=False)
341 group.add_option("", "--no-progress-bar", dest="useProgressBar",
342 help="Do not use curses based progress bar",
343 action="store_false", default=True)
344 parser.add_option_group(group)
346 group = OptionGroup(parser, "Test Execution")
347 group.add_option("", "--path", dest="path",
348 help="Additional paths to add to testing environment",
349 action="append", type=str, default=[])
350 group.add_option("", "--vg", dest="useValgrind",
351 help="Run tests under valgrind",
352 action="store_true", default=False)
353 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
354 help="Specify an extra argument for valgrind",
355 type=str, action="append", default=[])
356 group.add_option("", "--time-tests", dest="timeTests",
357 help="Track elapsed wall time for each test",
358 action="store_true", default=False)
359 group.add_option("", "--no-execute", dest="noExecute",
360 help="Don't execute any tests (assume PASS)",
361 action="store_true", default=False)
362 parser.add_option_group(group)
364 group = OptionGroup(parser, "Test Selection")
365 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
366 help="Maximum number of tests to run",
367 action="store", type=int, default=None)
368 group.add_option("", "--max-time", dest="maxTime", metavar="N",
369 help="Maximum time to spend testing (in seconds)",
370 action="store", type=float, default=None)
371 group.add_option("", "--shuffle", dest="shuffle",
372 help="Run tests in random order",
373 action="store_true", default=False)
374 parser.add_option_group(group)
376 group = OptionGroup(parser, "Debug and Experimental Options")
377 group.add_option("", "--debug", dest="debug",
378 help="Enable debugging (for 'lit' development)",
379 action="store_true", default=False)
380 group.add_option("", "--show-suites", dest="showSuites",
381 help="Show discovered test suites",
382 action="store_true", default=False)
383 group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
384 help="Don't run Tcl scripts using 'sh'",
385 action="store_false", default=True)
386 parser.add_option_group(group)
388 (opts, args) = parser.parse_args()
391 parser.error('No inputs specified')
393 if opts.configPrefix is not None:
394 global gConfigName, gSiteConfigName
395 gConfigName = '%s.cfg' % opts.configPrefix
396 gSiteConfigName = '%s.site.cfg' % opts.configPrefix
398 if opts.numThreads is None:
399 opts.numThreads = Util.detectCPUs()
403 # Create the user defined parameters.
405 for entry in opts.userParameters:
409 name,val = entry.split('=', 1)
410 userParams[name] = val
412 # Create the global config object.
413 litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
416 useValgrind = opts.useValgrind,
417 valgrindArgs = opts.valgrindArgs,
418 useTclAsSh = opts.useTclAsSh,
419 noExecute = opts.noExecute,
421 isWindows = (platform.system()=='Windows'),
424 # Load the tests from the inputs.
427 localConfigCache = {}
430 tests.extend(getTests(input, litConfig,
431 testSuiteCache, localConfigCache))
432 if prev == len(tests):
433 litConfig.warning('input %r contained no tests' % input)
435 # If there were any errors during test discovery, exit now.
436 if litConfig.numErrors:
437 print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
441 suitesAndTests = dict([(ts,[])
442 for ts,_ in testSuiteCache.values()
445 suitesAndTests[t.suite].append(t)
447 print '-- Test Suites --'
448 suitesAndTests = suitesAndTests.items()
449 suitesAndTests.sort(key = lambda (ts,_): ts.name)
450 for ts,tests in suitesAndTests:
451 print ' %s - %d tests' %(ts.name, len(tests))
452 print ' Source Root: %s' % ts.source_root
453 print ' Exec Root : %s' % ts.exec_root
455 # Select and order the tests.
456 numTotalTests = len(tests)
458 random.shuffle(tests)
460 tests.sort(key = lambda t: t.getFullName())
461 if opts.maxTests is not None:
462 tests = tests[:opts.maxTests]
465 if len(tests) != numTotalTests:
466 extra = ' of %d' % numTotalTests
467 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
472 if opts.succinct and opts.useProgressBar:
474 tc = ProgressBar.TerminalController()
475 progressBar = ProgressBar.ProgressBar(tc, header)
478 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
482 # Don't create more threads than tests.
483 opts.numThreads = min(len(tests), opts.numThreads)
485 startTime = time.time()
486 display = TestingProgressDisplay(opts, len(tests), progressBar)
487 provider = TestProvider(tests, opts.maxTime)
488 runTests(opts.numThreads, litConfig, provider, display)
492 print 'Testing Time: %.2fs'%(time.time() - startTime)
494 # Update results for any tests which weren't run.
497 t.setResult(Test.UNRESOLVED, '', 0.0)
499 # List test results organized by kind.
503 if t.result not in byCode:
504 byCode[t.result] = []
505 byCode[t.result].append(t)
506 if t.result.isFailure:
509 # FIXME: Show unresolved and (optionally) unsupported tests.
510 for title,code in (('Unexpected Passing Tests', Test.XPASS),
511 ('Failing Tests', Test.FAIL)):
512 elts = byCode.get(code)
516 print '%s (%d):' % (title, len(elts))
518 print ' %s' % t.getFullName()
523 byTime.sort(key = lambda t: t.elapsed)
525 Util.printHistogram([(t.getFullName(), t.elapsed) for t in byTime],
528 for name,code in (('Expected Passes ', Test.PASS),
529 ('Expected Failures ', Test.XFAIL),
530 ('Unsupported Tests ', Test.UNSUPPORTED),
531 ('Unresolved Tests ', Test.UNRESOLVED),
532 ('Unexpected Passes ', Test.XPASS),
533 ('Unexpected Failures', Test.FAIL),):
534 if opts.quiet and not code.isFailure:
536 N = len(byCode.get(code,[]))
538 print ' %s: %d' % (name,N)
540 # If we encountered any additional errors, exit abnormally.
541 if litConfig.numErrors:
542 print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
545 # Warn about warnings.
546 if litConfig.numWarnings:
547 print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
553 if __name__=='__main__':
554 # Bump the GIL check interval, its more important to get any one thread to a
555 # blocking operation (hopefully exec) than to try and unblock other threads.
557 sys.setcheckinterval(1000)