utils/lit/lit.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 import math, os, platform, random, re, sys, time, threading, traceback
  10
  11 import ProgressBar
  12 import TestRunner
  13 import Util
  14
  15 from TestingConfig import TestingConfig
  16 import LitConfig
  17 import Test
  18
  19 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
  20 kConfigName = 'lit.cfg'
  21 kSiteConfigName = 'lit.site.cfg'
  22 kLocalConfigName = 'lit.local.cfg'
  23
  24 class TestingProgressDisplay:
  25     def __init__(self, opts, numTests, progressBar=None):
  26         self.opts = opts
  27         self.numTests = numTests
  28         self.current = None
  29         self.lock = threading.Lock()
  30         self.progressBar = progressBar
  31         self.completed = 0
  32
  33     def update(self, test):
  34         # Avoid locking overhead in quiet mode
  35         if self.opts.quiet and not test.result.isFailure:
  36             return
  37
  38         # Output lock.
  39         self.lock.acquire()
  40         try:
  41             self.handleUpdate(test)
  42         finally:
  43             self.lock.release()
  44
  45     def finish(self):
  46         if self.progressBar:
  47             self.progressBar.clear()
  48         elif self.opts.quiet:
  49             pass
  50         elif self.opts.succinct:
  51             sys.stdout.write('\n')
  52
  53     def handleUpdate(self, test):
  54         self.completed += 1
  55         if self.progressBar:
  56             self.progressBar.update(float(self.completed)/self.numTests,
  57                                     test.getFullName())
  58
  59         if self.opts.succinct and not test.result.isFailure:
  60             return
  61
  62         if self.progressBar:
  63             self.progressBar.clear()
  64
  65         print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  66                                      self.completed, self.numTests)
  67
  68         if test.result.isFailure and self.opts.showOutput:
  69             print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  70                                               '*'*20)
  71             print test.output
  72             print "*" * 20
  73
  74         sys.stdout.flush()
  75
  76 class TestProvider:
  77     def __init__(self, tests, maxTime):
  78         self.maxTime = maxTime
  79         self.iter = iter(tests)
  80         self.lock = threading.Lock()
  81         self.startTime = time.time()
  82
  83     def get(self):
  84         # Check if we have run out of time.
  85         if self.maxTime is not None:
  86             if time.time() - self.startTime > self.maxTime:
  87                 return None
  88
  89         # Otherwise take the next test.
  90         self.lock.acquire()
  91         try:
  92             item = self.iter.next()
  93         except StopIteration:
  94             item = None
  95         self.lock.release()
  96         return item
  97
  98 class Tester(threading.Thread):
  99     def __init__(self, litConfig, provider, display):
 100         threading.Thread.__init__(self)
 101         self.litConfig = litConfig
 102         self.provider = provider
 103         self.display = display
 104
 105     def run(self):
 106         while 1:
 107             item = self.provider.get()
 108             if item is None:
 109                 break
 110             self.runTest(item)
 111
 112     def runTest(self, test):
 113         result = None
 114         startTime = time.time()
 115         try:
 116             result, output = test.config.test_format.execute(test,
 117                                                              self.litConfig)
 118         except KeyboardInterrupt:
 119             # This is a sad hack. Unfortunately subprocess goes
 120             # bonkers with ctrl-c and we start forking merrily.
 121             print '\nCtrl-C detected, goodbye.'
 122             os.kill(0,9)
 123         except:
 124             if self.litConfig.debug:
 125                 raise
 126             result = Test.UNRESOLVED
 127             output = 'Exception during script execution:\n'
 128             output += traceback.format_exc()
 129             output += '\n'
 130         elapsed = time.time() - startTime
 131
 132         test.setResult(result, output, elapsed)
 133         self.display.update(test)
 134
 135 def dirContainsTestSuite(path):
 136     cfgpath = os.path.join(path, kSiteConfigName)
 137     if os.path.exists(cfgpath):
 138         return cfgpath
 139     cfgpath = os.path.join(path, kConfigName)
 140     if os.path.exists(cfgpath):
 141         return cfgpath
 142
 143 def getTestSuite(item, litConfig, cache):
 144     """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
 145
 146     Find the test suite containing @arg item.
 147
 148     @retval (None, ...) - Indicates no test suite contains @arg item.
 149     @retval (suite, relative_path) - The suite that @arg item is in, and its
 150     relative path inside that suite.
 151     """
 152     def search1(path):
 153         # Check for a site config or a lit config.
 154         cfgpath = dirContainsTestSuite(path)
 155
 156         # If we didn't find a config file, keep looking.
 157         if not cfgpath:
 158             parent,base = os.path.split(path)
 159             if parent == path:
 160                 return (None, ())
 161
 162             ts, relative = search(parent)
 163             return (ts, relative + (base,))
 164
 165         # We found a config file, load it.
 166         if litConfig.debug:
 167             litConfig.note('loading suite config %r' % cfgpath)
 168
 169         cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
 170         source_root = os.path.realpath(cfg.test_source_root or path)
 171         exec_root = os.path.realpath(cfg.test_exec_root or path)
 172         return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
 173
 174     def search(path):
 175         # Check for an already instantiated test suite.
 176         res = cache.get(path)
 177         if res is None:
 178             cache[path] = res = search1(path)
 179         return res
 180
 181     # Canonicalize the path.
 182     item = os.path.realpath(item)
 183
 184     # Skip files and virtual components.
 185     components = []
 186     while not os.path.isdir(item):
 187         parent,base = os.path.split(item)
 188         if parent == item:
 189             return (None, ())
 190         components.append(base)
 191         item = parent
 192     components.reverse()
 193
 194     ts, relative = search(item)
 195     return ts, tuple(relative + tuple(components))
 196
 197 def getLocalConfig(ts, path_in_suite, litConfig, cache):
 198     def search1(path_in_suite):
 199         # Get the parent config.
 200         if not path_in_suite:
 201             parent = ts.config
 202         else:
 203             parent = search(path_in_suite[:-1])
 204
 205         # Load the local configuration.
 206         source_path = ts.getSourcePath(path_in_suite)
 207         cfgpath = os.path.join(source_path, kLocalConfigName)
 208         if litConfig.debug:
 209             litConfig.note('loading local config %r' % cfgpath)
 210         return TestingConfig.frompath(cfgpath, parent, litConfig,
 211                                     mustExist = False,
 212                                     config = parent.clone(cfgpath))
 213
 214     def search(path_in_suite):
 215         key = (ts, path_in_suite)
 216         res = cache.get(key)
 217         if res is None:
 218             cache[key] = res = search1(path_in_suite)
 219         return res
 220
 221     return search(path_in_suite)
 222
 223 def getTests(path, litConfig, testSuiteCache, localConfigCache):
 224     # Find the test suite for this input and its relative path.
 225     ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
 226     if ts is None:
 227         litConfig.warning('unable to find test suite for %r' % path)
 228         return ()
 229
 230     if litConfig.debug:
 231         litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
 232                                                         path_in_suite))
 233
 234     return getTestsInSuite(ts, path_in_suite, litConfig,
 235                            testSuiteCache, localConfigCache)
 236
 237 def getTestsInSuite(ts, path_in_suite, litConfig,
 238                     testSuiteCache, localConfigCache):
 239     # Check that the source path exists (errors here are reported by the
 240     # caller).
 241     source_path = ts.getSourcePath(path_in_suite)
 242     if not os.path.exists(source_path):
 243         return
 244
 245     # Check if the user named a test directly.
 246     if not os.path.isdir(source_path):
 247         lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
 248         yield Test.Test(ts, path_in_suite, lc)
 249         return
 250
 251     # Otherwise we have a directory to search for tests, start by getting the
 252     # local configuration.
 253     lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
 254     for filename in os.listdir(source_path):
 255         # FIXME: This doesn't belong here?
 256         if filename == 'Output' or filename in lc.excludes:
 257             continue
 258
 259         filepath = os.path.join(source_path, filename)
 260         if os.path.isdir(filepath):
 261             # If this directory contains a test suite, reload it.
 262             if dirContainsTestSuite(filepath):
 263                 for res in getTests(filepath, litConfig,
 264                                     testSuiteCache, localConfigCache):
 265                     yield res
 266             else:
 267                 # Otherwise, continue loading from inside this test suite.
 268                 for res in getTestsInSuite(ts, path_in_suite + (filename,),
 269                                            litConfig, testSuiteCache,
 270                                            localConfigCache):
 271                     yield res
 272         else:
 273             # Otherwise add tests for matching suffixes.
 274             base,ext = os.path.splitext(filename)
 275             if ext in lc.suffixes:
 276                 yield Test.Test(ts, path_in_suite + (filename,), lc)
 277
 278 def runTests(numThreads, litConfig, provider, display):
 279     # If only using one testing thread, don't use threads at all; this lets us
 280     # profile, among other things.
 281     if numThreads == 1:
 282         t = Tester(litConfig, provider, display)
 283         t.run()
 284         return
 285
 286     # Otherwise spin up the testing threads and wait for them to finish.
 287     testers = [Tester(litConfig, provider, display)
 288                for i in range(numThreads)]
 289     for t in testers:
 290         t.start()
 291     try:
 292         for t in testers:
 293             t.join()
 294     except KeyboardInterrupt:
 295         sys.exit(2)
 296
 297 def main():
 298     global options
 299     from optparse import OptionParser, OptionGroup
 300     parser = OptionParser("usage: %prog [options] {file-or-path}")
 301
 302     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 303                       help="Number of testing threads",
 304                       type=int, action="store", default=None)
 305
 306     group = OptionGroup(parser, "Output Format")
 307     # FIXME: I find these names very confusing, although I like the
 308     # functionality.
 309     group.add_option("-q", "--quiet", dest="quiet",
 310                      help="Suppress no error output",
 311                      action="store_true", default=False)
 312     group.add_option("-s", "--succinct", dest="succinct",
 313                      help="Reduce amount of output",
 314                      action="store_true", default=False)
 315     group.add_option("-v", "--verbose", dest="showOutput",
 316                      help="Show all test output",
 317                      action="store_true", default=False)
 318     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 319                      help="Do not use curses based progress bar",
 320                      action="store_false", default=True)
 321     parser.add_option_group(group)
 322
 323     group = OptionGroup(parser, "Test Execution")
 324     group.add_option("", "--path", dest="path",
 325                      help="Additional paths to add to testing environment",
 326                      action="append", type=str, default=[])
 327     group.add_option("", "--vg", dest="useValgrind",
 328                      help="Run tests under valgrind",
 329                      action="store_true", default=False)
 330     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 331                      help="Specify an extra argument for valgrind",
 332                      type=str, action="append", default=[])
 333     group.add_option("", "--time-tests", dest="timeTests",
 334                      help="Track elapsed wall time for each test",
 335                      action="store_true", default=False)
 336     group.add_option("", "--no-execute", dest="noExecute",
 337                      help="Don't execute any tests (assume PASS)",
 338                      action="store_true", default=False)
 339     parser.add_option_group(group)
 340
 341     group = OptionGroup(parser, "Test Selection")
 342     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 343                      help="Maximum number of tests to run",
 344                      action="store", type=int, default=None)
 345     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 346                      help="Maximum time to spend testing (in seconds)",
 347                      action="store", type=float, default=None)
 348     group.add_option("", "--shuffle", dest="shuffle",
 349                      help="Run tests in random order",
 350                      action="store_true", default=False)
 351     parser.add_option_group(group)
 352
 353     group = OptionGroup(parser, "Debug and Experimental Options")
 354     group.add_option("", "--debug", dest="debug",
 355                       help="Enable debugging (for 'lit' development)",
 356                       action="store_true", default=False)
 357     group.add_option("", "--show-suites", dest="showSuites",
 358                       help="Show discovered test suites",
 359                       action="store_true", default=False)
 360     group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
 361                       help="Don't run Tcl scripts using 'sh'",
 362                       action="store_false", default=True)
 363     parser.add_option_group(group)
 364
 365     (opts, args) = parser.parse_args()
 366
 367     if not args:
 368         parser.error('No inputs specified')
 369
 370     if opts.numThreads is None:
 371         opts.numThreads = Util.detectCPUs()
 372
 373     inputs = args
 374
 375     # Create the global config object.
 376     litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
 377                                     path = opts.path,
 378                                     quiet = opts.quiet,
 379                                     useValgrind = opts.useValgrind,
 380                                     valgrindArgs = opts.valgrindArgs,
 381                                     useTclAsSh = opts.useTclAsSh,
 382                                     noExecute = opts.noExecute,
 383                                     debug = opts.debug,
 384                                     isWindows = (platform.system()=='Windows'))
 385
 386     # Load the tests from the inputs.
 387     tests = []
 388     testSuiteCache = {}
 389     localConfigCache = {}
 390     for input in inputs:
 391         prev = len(tests)
 392         tests.extend(getTests(input, litConfig,
 393                               testSuiteCache, localConfigCache))
 394         if prev == len(tests):
 395             litConfig.warning('input %r contained no tests' % input)
 396
 397     # If there were any errors during test discovery, exit now.
 398     if litConfig.numErrors:
 399         print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
 400         sys.exit(2)
 401
 402     if opts.showSuites:
 403         suitesAndTests = dict([(ts,[])
 404                                for ts,_ in testSuiteCache.values()])
 405         for t in tests:
 406             suitesAndTests[t.suite].append(t)
 407
 408         print '-- Test Suites --'
 409         suitesAndTests = suitesAndTests.items()
 410         suitesAndTests.sort(key = lambda (ts,_): ts.name)
 411         for ts,tests in suitesAndTests:
 412             print '  %s - %d tests' %(ts.name, len(tests))
 413             print '    Source Root: %s' % ts.source_root
 414             print '    Exec Root  : %s' % ts.exec_root
 415
 416     # Select and order the tests.
 417     numTotalTests = len(tests)
 418     if opts.shuffle:
 419         random.shuffle(tests)
 420     else:
 421         tests.sort(key = lambda t: t.getFullName())
 422     if opts.maxTests is not None:
 423         tests = tests[:opts.maxTests]
 424
 425     extra = ''
 426     if len(tests) != numTotalTests:
 427         extra = ' of %d' % numTotalTests
 428     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 429                                                       opts.numThreads)
 430
 431     progressBar = None
 432     if not opts.quiet:
 433         if opts.succinct and opts.useProgressBar:
 434             try:
 435                 tc = ProgressBar.TerminalController()
 436                 progressBar = ProgressBar.ProgressBar(tc, header)
 437             except ValueError:
 438                 print header
 439                 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
 440         else:
 441             print header
 442
 443     # Don't create more threads than tests.
 444     opts.numThreads = min(len(tests), opts.numThreads)
 445
 446     startTime = time.time()
 447     display = TestingProgressDisplay(opts, len(tests), progressBar)
 448     provider = TestProvider(tests, opts.maxTime)
 449     runTests(opts.numThreads, litConfig, provider, display)
 450     display.finish()
 451
 452     if not opts.quiet:
 453         print 'Testing Time: %.2fs'%(time.time() - startTime)
 454
 455     # Update results for any tests which weren't run.
 456     for t in tests:
 457         if t.result is None:
 458             t.setResult(Test.UNRESOLVED, '', 0.0)
 459
 460     # List test results organized by kind.
 461     hasFailures = False
 462     byCode = {}
 463     for t in tests:
 464         if t.result not in byCode:
 465             byCode[t.result] = []
 466         byCode[t.result].append(t)
 467         if t.result.isFailure:
 468             hasFailures = True
 469
 470     # FIXME: Show unresolved and (optionally) unsupported tests.
 471     for title,code in (('Unexpected Passing Tests', Test.XPASS),
 472                        ('Failing Tests', Test.FAIL)):
 473         elts = byCode.get(code)
 474         if not elts:
 475             continue
 476         print '*'*20
 477         print '%s (%d):' % (title, len(elts))
 478         for t in elts:
 479             print '    %s' % t.getFullName()
 480         print
 481
 482     if opts.timeTests:
 483         byTime = list(tests)
 484         byTime.sort(key = lambda t: t.elapsed)
 485         if byTime:
 486             Util.printHistogram([(t.getFullName(), t.elapsed) for t in byTime],
 487                                 title='Tests')
 488
 489     for name,code in (('Expected Passes    ', Test.PASS),
 490                       ('Expected Failures  ', Test.XFAIL),
 491                       ('Unsupported Tests  ', Test.UNSUPPORTED),
 492                       ('Unresolved Tests   ', Test.UNRESOLVED),
 493                       ('Unexpected Passes  ', Test.XPASS),
 494                       ('Unexpected Failures', Test.FAIL),):
 495         if opts.quiet and not code.isFailure:
 496             continue
 497         N = len(byCode.get(code,[]))
 498         if N:
 499             print '  %s: %d' % (name,N)
 500
 501     # If we encountered any additional errors, exit abnormally.
 502     if litConfig.numErrors:
 503         print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
 504         sys.exit(2)
 505
 506     # Warn about warnings.
 507     if litConfig.numWarnings:
 508         print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
 509
 510     if hasFailures:
 511         sys.exit(1)
 512     sys.exit(0)
 513
 514 if __name__=='__main__':
 515     # Bump the GIL check interval, its more important to get any one thread to a
 516     # blocking operation (hopefully exec) than to try and unblock other threads.
 517     import sys
 518     sys.setcheckinterval(1000)
 519     main()