utils/llvm-compilers-check

   1 #!/usr/bin/python3
   2 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
   3 #
   4 #                     The LLVM Compiler Infrastructure
   5 #
   6 # This file is distributed under the University of Illinois Open Source
   7 # License. See LICENSE.TXT for details.
   8 #
   9 ##===----------------------------------------------------------------------===##
  10 #
  11 # This script builds many different flavors of the LLVM ecosystem.  It
  12 # will build LLVM, Clang and dragonegg as well as run tests on them.
  13 # This script is convenient to use to check builds and tests before
  14 # committing changes to the upstream repository
  15 #
  16 # A typical source setup uses three trees and looks like this:
  17 #
  18 # official
  19 #   dragonegg
  20 #   llvm
  21 #     tools
  22 #       clang
  23 # staging
  24 #   dragonegg
  25 #   llvm
  26 #     tools
  27 #       clang
  28 # commit
  29 #   dragonegg
  30 #   llvm
  31 #     tools
  32 #       clang
  33 #
  34 # In a typical workflow, the "official" tree always contains unchanged
  35 # sources from the main LLVM project repositories.  The "staging" tree
  36 # is where local work is done.  A set of changes resides there waiting
  37 # to be moved upstream.  The "commit" tree is where changes from
  38 # "staging" make their way upstream.  Individual incremental changes
  39 # from "staging" are applied to "commit" and committed upstream after
  40 # a successful build and test run.  A successful build is one in which
  41 # testing results in no more failures than seen in the testing of the
  42 # "official" tree.
  43 #
  44 # A build may be invoked as such:
  45 #
  46 # llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official
  47 #   --build=debug --build=release --build=paranoid
  48 #   --prefix=/home/greened/install --builddir=/home/greened/build
  49 #
  50 # This will build the LLVM ecosystem, including LLVM, Clangand
  51 # dragonegg, putting build results in ~/build and installing tools in
  52 # ~/install.  llvm-compilers-check creates separate build and install
  53 # directories for each source/build flavor.  In the above example,
  54 # llvmbuild will build debug, release and paranoid (debug+checks)
  55 # flavors from each source tree (official, staging and commit) for a
  56 # total of nine builds.  All builds will be run in parallel.
  57 #
  58 # The user may control parallelism via the --jobs and --threads
  59 # switches.  --jobs tells llvm-compilers-checl the maximum total
  60 # number of builds to activate in parallel.  The user may think of it
  61 # as equivalent to the GNU make -j switch.  --threads tells
  62 # llvm-compilers-check how many worker threads to use to accomplish
  63 # those builds.  If --threads is less than --jobs, --threads workers
  64 # will be launched and each one will pick a source/flavor combination
  65 # to build.  Then llvm-compilers-check will invoke GNU make with -j
  66 # (--jobs / --threads) to use up the remaining job capacity.  Once a
  67 # worker is finished with a build, it will pick another combination
  68 # off the list and start building it.
  69 #
  70 ##===----------------------------------------------------------------------===##
  71
  72 import optparse
  73 import os
  74 import sys
  75 import threading
  76 import queue
  77 import logging
  78 import traceback
  79 import subprocess
  80 import re
  81
  82 # TODO: Use shutil.which when it is available (3.2 or later)
  83 def find_executable(executable, path=None):
  84     """Try to find 'executable' in the directories listed in 'path' (a
  85     string listing directories separated by 'os.pathsep'; defaults to
  86     os.environ['PATH']).  Returns the complete filename or None if not
  87     found
  88     """
  89     if path is None:
  90         path = os.environ['PATH']
  91     paths = path.split(os.pathsep)
  92     extlist = ['']
  93     if os.name == 'os2':
  94         (base, ext) = os.path.splitext(executable)
  95         # executable files on OS/2 can have an arbitrary extension, but
  96         # .exe is automatically appended if no dot is present in the name
  97         if not ext:
  98             executable = executable + ".exe"
  99     elif sys.platform == 'win32':
 100         pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
 101         (base, ext) = os.path.splitext(executable)
 102         if ext.lower() not in pathext:
 103             extlist = pathext
 104     for ext in extlist:
 105         execname = executable + ext
 106         if os.path.isfile(execname):
 107             return execname
 108         else:
 109             for p in paths:
 110                 f = os.path.join(p, execname)
 111                 if os.path.isfile(f):
 112                     return f
 113     else:
 114         return None
 115
 116 def is_executable(fpath):
 117     return os.path.exists(fpath) and os.access(fpath, os.X_OK)
 118
 119 def add_options(parser):
 120     parser.add_option("-v", "--verbose", action="store_true",
 121                       default=False,
 122                       help=("Output informational messages"
 123                             " [default: %default]"))
 124     parser.add_option("--src", action="append",
 125                       help=("Top-level source directory [default: %default]"))
 126     parser.add_option("--build", action="append",
 127                       help=("Build types to run [default: %default]"))
 128     parser.add_option("--cc", default=find_executable("cc"),
 129                       help=("The C compiler to use [default: %default]"))
 130     parser.add_option("--cxx", default=find_executable("c++"),
 131                       help=("The C++ compiler to use [default: %default]"))
 132     parser.add_option("--threads", default=4, type="int",
 133                       help=("The number of worker threads to use "
 134                             "[default: %default]"))
 135     parser.add_option("--jobs", "-j", default=8, type="int",
 136                       help=("The number of simultaneous build jobs "
 137                             "[default: %default]"))
 138     parser.add_option("--prefix",
 139                       help=("Root install directory [default: %default]"))
 140     parser.add_option("--builddir",
 141                       help=("Root build directory [default: %default]"))
 142     parser.add_option("--extra-llvm-config-flags", default="",
 143                       help=("Extra flags to pass to llvm configure [default: %default]"))
 144     parser.add_option("--force-configure", default=False, action="store_true",
 145                       help=("Force reconfigure of all components"))
 146     parser.add_option("--no-dragonegg", default=False, action="store_true",
 147                       help=("Do not build dragonegg"))
 148     parser.add_option("--no-install", default=False, action="store_true",
 149                       help=("Do not do installs"))
 150     parser.add_option("--keep-going", default=False, action="store_true",
 151                       help=("Keep going after failures"))
 152     return
 153
 154 def check_options(parser, options, valid_builds):
 155     # See if we're building valid flavors.
 156     for build in options.build:
 157         if (build not in valid_builds):
 158             parser.error("'" + build + "' is not a valid build flavor "
 159                          + str(valid_builds))
 160
 161     # See if we can find source directories.
 162     for src in options.src:
 163         for component in components:
 164             component = component.rstrip("2")
 165             compsrc = src + "/" + component
 166             if (not os.path.isdir(compsrc)):
 167                 parser.error("'" + compsrc + "' does not exist")
 168
 169     # See if we can find the compilers
 170     options.cc = find_executable(options.cc)
 171     options.cxx = find_executable(options.cxx)
 172
 173     return
 174
 175 # Find a unique short name for the given set of paths.  This searches
 176 # back through path components until it finds unique component names
 177 # among all given paths.
 178 def get_path_abbrevs(paths):
 179     # Find the number of common starting characters in the last component
 180     # of the paths.
 181     unique_paths = list(paths)
 182
 183     class NotFoundException(Exception): pass
 184
 185     # Find a unique component of each path.
 186     unique_bases = unique_paths[:]
 187     found = 0
 188     while len(unique_paths) > 0:
 189         bases = [os.path.basename(src) for src in unique_paths]
 190         components = { c for c in bases }
 191         # Account for single entry in paths.
 192         if len(components) > 1 or len(components) == len(bases):
 193             # We found something unique.
 194             for c in components:
 195                 if bases.count(c) == 1:
 196                    index = bases.index(c)
 197                    unique_bases[index] = c
 198                    # Remove the corresponding path from the set under
 199                    # consideration.
 200                    unique_paths[index] = None
 201             unique_paths = [ p for p in unique_paths if p is not None ]
 202         unique_paths = [os.path.dirname(src) for src in unique_paths]
 203
 204     if len(unique_paths) > 0:
 205         raise NotFoundException()
 206
 207     abbrevs = dict(zip(paths, [base for base in unique_bases]))
 208
 209     return abbrevs
 210
 211 # Given a set of unique names, find a short character sequence that
 212 # uniquely identifies them.
 213 def get_short_abbrevs(unique_bases):
 214     # Find a unique start character for each path base.
 215     my_unique_bases = unique_bases[:]
 216     unique_char_starts = unique_bases[:]
 217     while len(my_unique_bases) > 0:
 218         for start, char_tuple in enumerate(zip(*[base
 219                                                  for base in my_unique_bases])):
 220             chars = { c for c in char_tuple }
 221             # Account for single path.
 222             if len(chars) > 1 or len(chars) == len(char_tuple):
 223                 # We found something unique.
 224                 for c in chars:
 225                     if char_tuple.count(c) == 1:
 226                         index = char_tuple.index(c)
 227                         unique_char_starts[index] = start
 228                         # Remove the corresponding path from the set under
 229                         # consideration.
 230                         my_unique_bases[index] = None
 231                 my_unique_bases = [ b for b in my_unique_bases
 232                                     if b is not None ]
 233                 break
 234
 235     if len(my_unique_bases) > 0:
 236         raise NotFoundException()
 237
 238     abbrevs = [abbrev[start_index:start_index+3]
 239                for abbrev, start_index
 240                in zip([base for base in unique_bases],
 241                       [index for index in unique_char_starts])]
 242
 243     abbrevs = dict(zip(unique_bases, abbrevs))
 244
 245     return abbrevs
 246
 247 class Builder(threading.Thread):
 248     class ExecutableNotFound(Exception): pass
 249     class FileNotExecutable(Exception): pass
 250
 251     def __init__(self, work_queue, jobs,
 252                  build_abbrev, source_abbrev,
 253                  options):
 254         super().__init__()
 255         self.work_queue = work_queue
 256         self.jobs = jobs
 257         self.cc = options.cc
 258         self.cxx = options.cxx
 259         self.build_abbrev = build_abbrev
 260         self.source_abbrev = source_abbrev
 261         self.build_prefix = options.builddir
 262         self.install_prefix = options.prefix
 263         self.options = options
 264         self.component_abbrev = dict(
 265             llvm="llvm",
 266             dragonegg="degg")
 267     def run(self):
 268         while True:
 269             try:
 270                 source, build = self.work_queue.get()
 271                 self.dobuild(source, build)
 272             except:
 273                 traceback.print_exc()
 274             finally:
 275                 self.work_queue.task_done()
 276
 277     def execute(self, command, execdir, env, component):
 278         prefix = self.component_abbrev[component.replace("-", "_")]
 279         pwd = os.getcwd()
 280         if not os.path.exists(execdir):
 281             os.makedirs(execdir)
 282
 283         execenv = os.environ.copy()
 284
 285         for key, value in env.items():
 286             execenv[key] = value
 287
 288         self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
 289                           + " ".join(command));
 290
 291         try:
 292             proc = subprocess.Popen(command,
 293                                     cwd=execdir,
 294                                     env=execenv,
 295                                     stdout=subprocess.PIPE,
 296                                     stderr=subprocess.STDOUT)
 297
 298             line = proc.stdout.readline()
 299             while line:
 300                 self.logger.info("[" + prefix + "] "
 301                                  + str(line, "utf-8").rstrip())
 302                 line = proc.stdout.readline()
 303
 304             (stdoutdata, stderrdata) = proc.communicate()
 305             retcode = proc.wait()
 306
 307             return retcode
 308
 309         except:
 310             traceback.print_exc()
 311
 312     # Get a list of C++ include directories to pass to clang.
 313     def get_includes(self):
 314         # Assume we're building with g++ for now.
 315         command = [self.cxx]
 316         command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
 317         includes = []
 318         self.logger.debug(command)
 319         try:
 320             proc = subprocess.Popen(command,
 321                                     stdout=subprocess.PIPE,
 322                                     stderr=subprocess.STDOUT)
 323
 324             gather = False
 325             line = proc.stdout.readline()
 326             while line:
 327                 self.logger.debug(line)
 328                 if re.search("End of search list", str(line)) is not None:
 329                     self.logger.debug("Stop Gather")
 330                     gather = False
 331                 if gather:
 332                     includes.append(str(line, "utf-8").strip())
 333                 if re.search("#include <...> search starts", str(line)) is not None:
 334                     self.logger.debug("Start Gather")
 335                     gather = True
 336                 line = proc.stdout.readline()
 337
 338         except:
 339             traceback.print_exc()
 340         self.logger.debug(includes)
 341         return includes
 342
 343     def dobuild(self, source, build):
 344         build_suffix = ""
 345
 346         ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
 347
 348         prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
 349         self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
 350         build_suffix += "/" + self.source_abbrev[source] + "/" + build
 351
 352         self.logger = logging.getLogger(prefix)
 353
 354         self.logger.debug(self.install_prefix)
 355
 356         # Assume we're building with gcc for now.
 357         cxxincludes = self.get_includes()
 358         cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version
 359         cxxroot = os.path.dirname(cxxroot)        # Remove the c++
 360         cxxroot = os.path.dirname(cxxroot)        # Remove the include
 361
 362         configure_flags = dict(
 363             llvm=dict(debug=["--prefix=" + self.install_prefix,
 364                              "--enable-werror",
 365                              "--enable-assertions",
 366                              "--disable-optimized",
 367                              "--with-gcc-toolchain=" + cxxroot],
 368                       release=["--prefix=" + self.install_prefix,
 369                                "--enable-werror",
 370                                "--enable-optimized",
 371                                "--with-gcc-toolchain=" + cxxroot],
 372                       paranoid=["--prefix=" + self.install_prefix,
 373                                 "--enable-werror",
 374                                 "--enable-assertions",
 375                                 "--enable-expensive-checks",
 376                                 "--disable-optimized",
 377                                 "--with-gcc-toolchain=" + cxxroot]),
 378             dragonegg=dict(debug=[],
 379                            release=[],
 380                            paranoid=[]))
 381
 382         configure_env = dict(
 383             llvm=dict(debug=dict(CC=self.cc,
 384                                  CXX=self.cxx),
 385                       release=dict(CC=self.cc,
 386                                    CXX=self.cxx),
 387                       paranoid=dict(CC=self.cc,
 388                                     CXX=self.cxx)),
 389             dragonegg=dict(debug=dict(CC=self.cc,
 390                                       CXX=self.cxx),
 391                            release=dict(CC=self.cc,
 392                                         CXX=self.cxx),
 393                            paranoid=dict(CC=self.cc,
 394                                          CXX=self.cxx)))
 395
 396         make_flags = dict(
 397             llvm=dict(debug=["-j" + str(self.jobs)],
 398                       release=["-j" + str(self.jobs)],
 399                       paranoid=["-j" + str(self.jobs)]),
 400             dragonegg=dict(debug=["-j" + str(self.jobs)],
 401                            release=["-j" + str(self.jobs)],
 402                            paranoid=["-j" + str(self.jobs)]))
 403
 404         make_env = dict(
 405             llvm=dict(debug=dict(),
 406                       release=dict(),
 407                       paranoid=dict()),
 408             dragonegg=dict(debug=dict(GCC=self.cc,
 409                                       LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
 410                            release=dict(GCC=self.cc,
 411                                         LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
 412                            paranoid=dict(GCC=self.cc,
 413                                          LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
 414
 415         make_install_flags = dict(
 416             llvm=dict(debug=["install"],
 417                       release=["install"],
 418                       paranoid=["install"]),
 419             dragonegg=dict(debug=["install"],
 420                            release=["install"],
 421                            paranoid=["install"]))
 422
 423         make_install_env = dict(
 424             llvm=dict(debug=dict(),
 425                       release=dict(),
 426                       paranoid=dict()),
 427             dragonegg=dict(debug=dict(),
 428                            release=dict(),
 429                            paranoid=dict()))
 430
 431         make_check_flags = dict(
 432             llvm=dict(debug=["check"],
 433                       release=["check"],
 434                       paranoid=["check"]),
 435             dragonegg=dict(debug=["check"],
 436                            release=["check"],
 437                            paranoid=["check"]))
 438
 439         make_check_env = dict(
 440             llvm=dict(debug=dict(),
 441                       release=dict(),
 442                       paranoid=dict()),
 443             dragonegg=dict(debug=dict(),
 444                            release=dict(),
 445                            paranoid=dict()))
 446
 447         for component in components:
 448             comp = component[:]
 449
 450             if (self.options.no_dragonegg):
 451                 if (comp == 'dragonegg'):
 452                     self.logger.info("Skipping " + component + " in "
 453                                      + builddir)
 454                     continue
 455
 456             srcdir = source + "/" + comp.rstrip("2")
 457             builddir = self.build_prefix + "/" + comp + "/" + build_suffix
 458             installdir = self.install_prefix
 459
 460             comp_key = comp.replace("-", "_")
 461
 462             config_args = configure_flags[comp_key][build][:]
 463             config_args.extend(getattr(self.options,
 464                                        "extra_" + comp_key.rstrip("2")
 465                                        + "_config_flags",
 466                                        "").split())
 467
 468             self.logger.info("Configuring " + component + " in " + builddir)
 469             configrc = self.configure(component, srcdir, builddir,
 470                                       config_args,
 471                                       configure_env[comp_key][build])
 472
 473             if (configrc == None) :
 474                 self.logger.info("[None] Failed to configure " + component + " in " + installdir)
 475
 476             if (configrc == 0 or self.options.keep_going) :
 477                 self.logger.info("Building " + component + " in " + builddir)
 478                 self.logger.info("Build: make " + str(make_flags[comp_key][build]))
 479                 buildrc = self.make(component, srcdir, builddir,
 480                                     make_flags[comp_key][build],
 481                                     make_env[comp_key][build])
 482
 483                 if (buildrc == None) :
 484                     self.logger.info("[None] Failed to build " + component + " in " + installdir)
 485
 486                 if (buildrc == 0 or self.options.keep_going) :
 487                     self.logger.info("Testing " + component + " in " + builddir)
 488                     self.logger.info("Test: make "
 489                                      + str(make_check_flags[comp_key][build]))
 490                     testrc = self.make(component, srcdir, builddir,
 491                                        make_check_flags[comp_key][build],
 492                                        make_check_env[comp_key][build])
 493
 494                     if (testrc == None) :
 495                         self.logger.info("[None] Failed to test " + component + " in " + installdir)
 496
 497                     if ((testrc == 0  or self.options.keep_going)
 498                         and not self.options.no_install):
 499                         self.logger.info("Installing " + component + " in " + installdir)
 500                         self.make(component, srcdir, builddir,
 501                                   make_install_flags[comp_key][build],
 502                                   make_install_env[comp_key][build])
 503                     else :
 504                         self.logger.info("Failed testing " + component + " in " + installdir)
 505
 506                 else :
 507                     self.logger.info("Failed to build " + component + " in " + installdir)
 508
 509             else :
 510                 self.logger.info("Failed to configure " + component + " in " + installdir)
 511
 512     def configure(self, component, srcdir, builddir, flags, env):
 513         prefix = self.component_abbrev[component.replace("-", "_")]
 514
 515         self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
 516                           + str(builddir))
 517
 518         configure_files = dict(
 519             llvm=[(srcdir + "/configure", builddir + "/Makefile")],
 520             dragonegg=[(None,None)])
 521
 522
 523         doconfig = False
 524         for conf, mf in configure_files[component.replace("-", "_")]:
 525             if conf is None:
 526                 # No configure necessary
 527                 return 0
 528
 529             if not os.path.exists(conf):
 530                 self.logger.info("[" + prefix + "] Configure failed, no configure script " + conf)
 531                 return -1
 532
 533             if not os.path.exists(mf):
 534                 self.logger.info("[" + prefix + "] Configure failed, no makefile " + mf)
 535                 return -1
 536
 537             if os.path.exists(conf) and os.path.exists(mf):
 538                 confstat = os.stat(conf)
 539                 makestat = os.stat(mf)
 540                 if confstat.st_mtime > makestat.st_mtime:
 541                     doconfig = True
 542                     break
 543             else:
 544                 doconfig = True
 545                 break
 546
 547         if not doconfig and not self.options.force_configure:
 548             return 0
 549
 550         program = srcdir + "/configure"
 551         if not is_executable(program):
 552             self.logger.info("[" + prefix + "] Configure failed, cannot execute " + program)
 553             return -1
 554
 555         args = [program]
 556         args += ["--verbose"]
 557         args += flags
 558         return self.execute(args, builddir, env, component)
 559
 560     def make(self, component, srcdir, builddir, flags, env):
 561         program = find_executable("make")
 562         if program is None:
 563             raise ExecutableNotFound
 564
 565         if not is_executable(program):
 566             raise FileNotExecutable
 567
 568         args = [program]
 569         args += flags
 570         return self.execute(args, builddir, env, component)
 571
 572 # Global constants
 573 build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
 574 components = ["llvm", "dragonegg"]
 575
 576 # Parse options
 577 parser = optparse.OptionParser(version="%prog 1.0")
 578 add_options(parser)
 579 (options, args) = parser.parse_args()
 580 check_options(parser, options, build_abbrev.keys());
 581
 582 if options.verbose:
 583     logging.basicConfig(level=logging.DEBUG,
 584                         format='%(name)-13s: %(message)s')
 585 else:
 586     logging.basicConfig(level=logging.INFO,
 587                         format='%(name)-13s: %(message)s')
 588
 589 source_abbrev = get_path_abbrevs(set(options.src))
 590
 591 work_queue = queue.Queue()
 592
 593 jobs = options.jobs // options.threads
 594 if jobs == 0:
 595     jobs = 1
 596
 597 numthreads = options.threads
 598
 599 logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
 600                          + str(numthreads) + " threads using " + str(jobs)
 601                          + " make jobs")
 602
 603 logging.getLogger().info("CC  = " + str(options.cc))
 604 logging.getLogger().info("CXX = " + str(options.cxx))
 605
 606 for t in range(numthreads):
 607     builder = Builder(work_queue, jobs,
 608                       build_abbrev, source_abbrev,
 609                       options)
 610     builder.daemon = True
 611     builder.start()
 612
 613 for build in set(options.build):
 614     for source in set(options.src):
 615         work_queue.put((source, build))
 616
 617 work_queue.join()