utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 import math, os, platform, random, re, sys, time, threading, traceback
  10
  11 import ProgressBar
  12 import TestRunner
  13 import Util
  14
  15 import LitConfig
  16 import Test
  17
  18 import lit.discovery
  19
  20 class TestingProgressDisplay:
  21     def __init__(self, opts, numTests, progressBar=None):
  22         self.opts = opts
  23         self.numTests = numTests
  24         self.current = None
  25         self.lock = threading.Lock()
  26         self.progressBar = progressBar
  27         self.completed = 0
  28
  29     def update(self, test):
  30         # Avoid locking overhead in quiet mode
  31         if self.opts.quiet and not test.result.isFailure:
  32             self.completed += 1
  33             return
  34
  35         # Output lock.
  36         self.lock.acquire()
  37         try:
  38             self.handleUpdate(test)
  39         finally:
  40             self.lock.release()
  41
  42     def finish(self):
  43         if self.progressBar:
  44             self.progressBar.clear()
  45         elif self.opts.quiet:
  46             pass
  47         elif self.opts.succinct:
  48             sys.stdout.write('\n')
  49
  50     def handleUpdate(self, test):
  51         self.completed += 1
  52         if self.progressBar:
  53             self.progressBar.update(float(self.completed)/self.numTests,
  54                                     test.getFullName())
  55
  56         if self.opts.succinct and not test.result.isFailure:
  57             return
  58
  59         if self.progressBar:
  60             self.progressBar.clear()
  61
  62         print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  63                                      self.completed, self.numTests)
  64
  65         if test.result.isFailure and self.opts.showOutput:
  66             print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  67                                               '*'*20)
  68             print test.output
  69             print "*" * 20
  70
  71         sys.stdout.flush()
  72
  73 class TestProvider:
  74     def __init__(self, tests, maxTime):
  75         self.maxTime = maxTime
  76         self.iter = iter(tests)
  77         self.lock = threading.Lock()
  78         self.startTime = time.time()
  79         self.canceled = False
  80
  81     def cancel(self):
  82         self.lock.acquire()
  83         self.canceled = True
  84         self.lock.release()
  85
  86     def get(self):
  87         # Check if we have run out of time.
  88         if self.maxTime is not None:
  89             if time.time() - self.startTime > self.maxTime:
  90                 return None
  91
  92         # Otherwise take the next test.
  93         self.lock.acquire()
  94         if self.canceled:
  95           self.lock.release()
  96           return None
  97
  98         try:
  99             item = self.iter.next()
 100         except StopIteration:
 101             item = None
 102         self.lock.release()
 103         return item
 104
 105 class Tester(threading.Thread):
 106     def __init__(self, litConfig, provider, display):
 107         threading.Thread.__init__(self)
 108         self.litConfig = litConfig
 109         self.provider = provider
 110         self.display = display
 111
 112     def run(self):
 113         while 1:
 114             item = self.provider.get()
 115             if item is None:
 116                 break
 117             self.runTest(item)
 118
 119     def runTest(self, test):
 120         result = None
 121         startTime = time.time()
 122         try:
 123             result, output = test.config.test_format.execute(test,
 124                                                              self.litConfig)
 125         except KeyboardInterrupt:
 126             # This is a sad hack. Unfortunately subprocess goes
 127             # bonkers with ctrl-c and we start forking merrily.
 128             print '\nCtrl-C detected, goodbye.'
 129             os.kill(0,9)
 130         except:
 131             if self.litConfig.debug:
 132                 raise
 133             result = Test.UNRESOLVED
 134             output = 'Exception during script execution:\n'
 135             output += traceback.format_exc()
 136             output += '\n'
 137         elapsed = time.time() - startTime
 138
 139         test.setResult(result, output, elapsed)
 140         self.display.update(test)
 141
 142 def runTests(numThreads, litConfig, provider, display):
 143     # If only using one testing thread, don't use threads at all; this lets us
 144     # profile, among other things.
 145     if numThreads == 1:
 146         t = Tester(litConfig, provider, display)
 147         t.run()
 148         return
 149
 150     # Otherwise spin up the testing threads and wait for them to finish.
 151     testers = [Tester(litConfig, provider, display)
 152                for i in range(numThreads)]
 153     for t in testers:
 154         t.start()
 155     try:
 156         for t in testers:
 157             t.join()
 158     except KeyboardInterrupt:
 159         sys.exit(2)
 160
 161 def main(builtinParameters = {}):
 162     # Bump the GIL check interval, its more important to get any one thread to a
 163     # blocking operation (hopefully exec) than to try and unblock other threads.
 164     #
 165     # FIXME: This is a hack.
 166     import sys
 167     sys.setcheckinterval(1000)
 168
 169     global options
 170     from optparse import OptionParser, OptionGroup
 171     parser = OptionParser("usage: %prog [options] {file-or-path}")
 172
 173     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 174                       help="Number of testing threads",
 175                       type=int, action="store", default=None)
 176     parser.add_option("", "--config-prefix", dest="configPrefix",
 177                       metavar="NAME", help="Prefix for 'lit' config files",
 178                       action="store", default=None)
 179     parser.add_option("", "--param", dest="userParameters",
 180                       metavar="NAME=VAL",
 181                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 182                       type=str, action="append", default=[])
 183
 184     group = OptionGroup(parser, "Output Format")
 185     # FIXME: I find these names very confusing, although I like the
 186     # functionality.
 187     group.add_option("-q", "--quiet", dest="quiet",
 188                      help="Suppress no error output",
 189                      action="store_true", default=False)
 190     group.add_option("-s", "--succinct", dest="succinct",
 191                      help="Reduce amount of output",
 192                      action="store_true", default=False)
 193     group.add_option("-v", "--verbose", dest="showOutput",
 194                      help="Show all test output",
 195                      action="store_true", default=False)
 196     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 197                      help="Do not use curses based progress bar",
 198                      action="store_false", default=True)
 199     parser.add_option_group(group)
 200
 201     group = OptionGroup(parser, "Test Execution")
 202     group.add_option("", "--path", dest="path",
 203                      help="Additional paths to add to testing environment",
 204                      action="append", type=str, default=[])
 205     group.add_option("", "--vg", dest="useValgrind",
 206                      help="Run tests under valgrind",
 207                      action="store_true", default=False)
 208     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 209                      help="Check for memory leaks under valgrind",
 210                      action="store_true", default=False)
 211     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 212                      help="Specify an extra argument for valgrind",
 213                      type=str, action="append", default=[])
 214     group.add_option("", "--time-tests", dest="timeTests",
 215                      help="Track elapsed wall time for each test",
 216                      action="store_true", default=False)
 217     group.add_option("", "--no-execute", dest="noExecute",
 218                      help="Don't execute any tests (assume PASS)",
 219                      action="store_true", default=False)
 220     parser.add_option_group(group)
 221
 222     group = OptionGroup(parser, "Test Selection")
 223     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 224                      help="Maximum number of tests to run",
 225                      action="store", type=int, default=None)
 226     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 227                      help="Maximum time to spend testing (in seconds)",
 228                      action="store", type=float, default=None)
 229     group.add_option("", "--shuffle", dest="shuffle",
 230                      help="Run tests in random order",
 231                      action="store_true", default=False)
 232     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 233                      help=("Only run tests with paths matching the given "
 234                            "regular expression"),
 235                      action="store", default=None)
 236     parser.add_option_group(group)
 237
 238     group = OptionGroup(parser, "Debug and Experimental Options")
 239     group.add_option("", "--debug", dest="debug",
 240                       help="Enable debugging (for 'lit' development)",
 241                       action="store_true", default=False)
 242     group.add_option("", "--show-suites", dest="showSuites",
 243                       help="Show discovered test suites",
 244                       action="store_true", default=False)
 245     group.add_option("", "--repeat", dest="repeatTests", metavar="N",
 246                       help="Repeat tests N times (for timing)",
 247                       action="store", default=None, type=int)
 248     parser.add_option_group(group)
 249
 250     (opts, args) = parser.parse_args()
 251
 252     if not args:
 253         parser.error('No inputs specified')
 254
 255     if opts.numThreads is None:
 256 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 257 # http://bugs.python.org/issue1731717
 258 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 259 # threads by default there.
 260        if sys.hexversion >= 0x2050200:
 261                opts.numThreads = Util.detectCPUs()
 262        else:
 263                opts.numThreads = 1
 264
 265     inputs = args
 266
 267     # Create the user defined parameters.
 268     userParams = dict(builtinParameters)
 269     for entry in opts.userParameters:
 270         if '=' not in entry:
 271             name,val = entry,''
 272         else:
 273             name,val = entry.split('=', 1)
 274         userParams[name] = val
 275
 276     # Create the global config object.
 277     litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
 278                                     path = opts.path,
 279                                     quiet = opts.quiet,
 280                                     useValgrind = opts.useValgrind,
 281                                     valgrindLeakCheck = opts.valgrindLeakCheck,
 282                                     valgrindArgs = opts.valgrindArgs,
 283                                     noExecute = opts.noExecute,
 284                                     debug = opts.debug,
 285                                     isWindows = (platform.system()=='Windows'),
 286                                     params = userParams,
 287                                     config_prefix = opts.configPrefix)
 288
 289     tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
 290
 291     if opts.showSuites:
 292         suitesAndTests = {}
 293         for t in tests:
 294             if t.suite not in suitesAndTests:
 295                 suitesAndTests[t.suite] = []
 296             suitesAndTests[t.suite].append(t)
 297
 298         print '-- Test Suites --'
 299         suitesAndTests = suitesAndTests.items()
 300         suitesAndTests.sort(key = lambda (ts,_): ts.name)
 301         for ts,ts_tests in suitesAndTests:
 302             print '  %s - %d tests' %(ts.name, len(ts_tests))
 303             print '    Source Root: %s' % ts.source_root
 304             print '    Exec Root  : %s' % ts.exec_root
 305
 306     # Select and order the tests.
 307     numTotalTests = len(tests)
 308
 309     # First, select based on the filter expression if given.
 310     if opts.filter:
 311         try:
 312             rex = re.compile(opts.filter)
 313         except:
 314             parser.error("invalid regular expression for --filter: %r" % (
 315                     opts.filter))
 316         tests = [t for t in tests
 317                  if rex.search(t.getFullName())]
 318
 319     # Then select the order.
 320     if opts.shuffle:
 321         random.shuffle(tests)
 322     else:
 323         tests.sort(key = lambda t: t.getFullName())
 324
 325     # Finally limit the number of tests, if desired.
 326     if opts.maxTests is not None:
 327         tests = tests[:opts.maxTests]
 328
 329     # Don't create more threads than tests.
 330     opts.numThreads = min(len(tests), opts.numThreads)
 331
 332     extra = ''
 333     if len(tests) != numTotalTests:
 334         extra = ' of %d' % numTotalTests
 335     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 336                                                       opts.numThreads)
 337
 338     if opts.repeatTests:
 339         tests = [t.copyWithIndex(i)
 340                  for t in tests
 341                  for i in range(opts.repeatTests)]
 342
 343     progressBar = None
 344     if not opts.quiet:
 345         if opts.succinct and opts.useProgressBar:
 346             try:
 347                 tc = ProgressBar.TerminalController()
 348                 progressBar = ProgressBar.ProgressBar(tc, header)
 349             except ValueError:
 350                 print header
 351                 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
 352         else:
 353             print header
 354
 355     startTime = time.time()
 356     display = TestingProgressDisplay(opts, len(tests), progressBar)
 357     provider = TestProvider(tests, opts.maxTime)
 358
 359     try:
 360       import win32api
 361     except ImportError:
 362       pass
 363     else:
 364       def console_ctrl_handler(type):
 365         provider.cancel()
 366         return True
 367       win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
 368
 369     runTests(opts.numThreads, litConfig, provider, display)
 370     display.finish()
 371
 372     if not opts.quiet:
 373         print 'Testing Time: %.2fs'%(time.time() - startTime)
 374
 375     # Update results for any tests which weren't run.
 376     for t in tests:
 377         if t.result is None:
 378             t.setResult(Test.UNRESOLVED, '', 0.0)
 379
 380     # List test results organized by kind.
 381     hasFailures = False
 382     byCode = {}
 383     for t in tests:
 384         if t.result not in byCode:
 385             byCode[t.result] = []
 386         byCode[t.result].append(t)
 387         if t.result.isFailure:
 388             hasFailures = True
 389
 390     # FIXME: Show unresolved and (optionally) unsupported tests.
 391     for title,code in (('Unexpected Passing Tests', Test.XPASS),
 392                        ('Failing Tests', Test.FAIL)):
 393         elts = byCode.get(code)
 394         if not elts:
 395             continue
 396         print '*'*20
 397         print '%s (%d):' % (title, len(elts))
 398         for t in elts:
 399             print '    %s' % t.getFullName()
 400         print
 401
 402     if opts.timeTests:
 403         # Collate, in case we repeated tests.
 404         times = {}
 405         for t in tests:
 406             key = t.getFullName()
 407             times[key] = times.get(key, 0.) + t.elapsed
 408
 409         byTime = list(times.items())
 410         byTime.sort(key = lambda (name,elapsed): elapsed)
 411         if byTime:
 412             Util.printHistogram(byTime, title='Tests')
 413
 414     for name,code in (('Expected Passes    ', Test.PASS),
 415                       ('Expected Failures  ', Test.XFAIL),
 416                       ('Unsupported Tests  ', Test.UNSUPPORTED),
 417                       ('Unresolved Tests   ', Test.UNRESOLVED),
 418                       ('Unexpected Passes  ', Test.XPASS),
 419                       ('Unexpected Failures', Test.FAIL),):
 420         if opts.quiet and not code.isFailure:
 421             continue
 422         N = len(byCode.get(code,[]))
 423         if N:
 424             print '  %s: %d' % (name,N)
 425
 426     # If we encountered any additional errors, exit abnormally.
 427     if litConfig.numErrors:
 428         print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
 429         sys.exit(2)
 430
 431     # Warn about warnings.
 432     if litConfig.numWarnings:
 433         print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
 434
 435     if hasFailures:
 436         sys.exit(1)
 437     sys.exit(0)
 438
 439 if __name__=='__main__':
 440     main()