2 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
4 # The LLVM Compiler Infrastructure
6 # This file is distributed under the University of Illinois Open Source
7 # License. See LICENSE.TXT for details.
9 ##===----------------------------------------------------------------------===##
11 # This script builds many different flavors of the LLVM ecosystem. It
12 # will build LLVM, Clang, llvm-gcc, and dragonegg as well as run tests
13 # on them. This script is convenient to use to check builds and tests
14 # before committing changes to the upstream repository
16 # A typical source setup uses three trees and looks like this:
70 # "gcc" above is the upstream FSF gcc and "gcc/trunk" refers to the
71 # 4.5 branch as discussed in the dragonegg build guide.
73 # In a typical workflow, the "official" tree always contains unchanged
74 # sources from the main LLVM project repositories. The "staging" tree
75 # is where local work is done. A set of changes resides there waiting
76 # to be moved upstream. The "commit" tree is where changes from
77 # "staging" make their way upstream. Individual incremental changes
78 # from "staging" are applied to "commit" and committed upstream after
79 # a successful build and test run. A successful build is one in which
80 # testing results in no more failures than seen in the testing of the
83 # A build may be invoked as such:
85 # llvmbuild --src=~/llvm/commit --src=~/llvm/staging
86 # --src=~/llvm/official --branch=trunk --branch=tags/RELEASE_28
87 # --build=debug --build=release --build=paranoid
88 # --prefix=/home/greened/install --builddir=/home/greened/build
90 # This will build the LLVM ecosystem, including LLVM, Clang, llvm-gcc,
91 # gcc 4.5 and dragonegg, putting build results in ~/build and
92 # installing tools in ~/install. llvmbuild creates separate build and
93 # install directories for each source/branch/build flavor. In the
94 # above example, llvmbuild will build debug, release and paranoid
95 # (debug+checks) flavors of the trunk and RELEASE_28 branches from
96 # each source tree (official, staging and commit) for a total of
97 # eighteen builds. All builds will be run in parallel.
99 # The user may control parallelism via the --jobs and --threads
100 # switches. --jobs tells llvmbuild the maximum total number of builds
101 # to activate in parallel. The user may think of it as equivalent to
102 # the GNU make -j switch. --threads tells llvmbuild how many worker
103 # threads to use to accomplish those builds. If --threads is less
104 # than --jobs, --threads workers will be launched and each one will
105 # pick a source/branch/flavor combination to build. Then llvmbuild
106 # will invoke GNU make with -j (--jobs / --threads) to use up the
107 # remaining job capacity. Once a worker is finished with a build, it
108 # will pick another combination off the list and start building it.
110 ##===----------------------------------------------------------------------===##
122 # TODO: Use shutil.which when it is available (3.2 or later)
123 def find_executable(executable, path=None):
124 """Try to find 'executable' in the directories listed in 'path' (a
125 string listing directories separated by 'os.pathsep'; defaults to
126 os.environ['PATH']). Returns the complete filename or None if not
130 path = os.environ['PATH']
131 paths = path.split(os.pathsep)
134 (base, ext) = os.path.splitext(executable)
135 # executable files on OS/2 can have an arbitrary extension, but
136 # .exe is automatically appended if no dot is present in the name
138 executable = executable + ".exe"
139 elif sys.platform == 'win32':
140 pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
141 (base, ext) = os.path.splitext(executable)
142 if ext.lower() not in pathext:
145 execname = executable + ext
146 if os.path.isfile(execname):
150 f = os.path.join(p, execname)
151 if os.path.isfile(f):
156 def is_executable(fpath):
157 return os.path.exists(fpath) and os.access(fpath, os.X_OK)
159 def add_options(parser):
160 parser.add_option("-v", "--verbose", action="store_true",
162 help=("Output informational messages"
163 " [default: %default]"))
164 parser.add_option("--src", action="append",
165 help=("Top-level source directory [default: %default]"))
166 parser.add_option("--build", action="append",
167 help=("Build types to run [default: %default]"))
168 parser.add_option("--branch", action="append",
169 help=("Source branch to build [default: %default]"))
170 parser.add_option("--cc", default=find_executable("cc"),
171 help=("The C compiler to use [default: %default]"))
172 parser.add_option("--cxx", default=find_executable("c++"),
173 help=("The C++ compiler to use [default: %default]"))
174 parser.add_option("--threads", default=4, type="int",
175 help=("The number of worker threads to use "
176 "[default: %default]"))
177 parser.add_option("--jobs", "-j", default=8, type="int",
178 help=("The number of simultaneous build jobs "
179 "[default: %default]"))
180 parser.add_option("--prefix",
181 help=("Root install directory [default: %default]"))
182 parser.add_option("--builddir",
183 help=("Root build directory [default: %default]"))
184 parser.add_option("--extra-llvm-config-flags", default="",
185 help=("Extra flags to pass to llvm configure [default: %default]"))
186 parser.add_option("--extra-llvm-gcc-config-flags", default="",
187 help=("Extra flags to pass to llvm-gcc configure [default: %default]"))
188 parser.add_option("--extra-gcc-config-flags", default="",
189 help=("Extra flags to pass to gcc configure [default: %default]"))
190 parser.add_option("--force-configure", default=False, action="store_true",
191 help=("Force reconfigure of all components"))
192 parser.add_option("--no-gcc", default=False, action="store_true",
193 help=("Do not build dragonegg and gcc"))
194 parser.add_option("--no-install", default=False, action="store_true",
195 help=("Do not do installs"))
198 def check_options(parser, options, valid_builds):
199 # See if we're building valid flavors.
200 for build in options.build:
201 if (build not in valid_builds):
202 parser.error("'" + build + "' is not a valid build flavor "
205 # See if we can find source directories.
206 for src in options.src:
207 for component in components:
208 component = component.rstrip("2")
209 compsrc = src + "/" + component
210 if (not os.path.isdir(compsrc)):
211 parser.error("'" + compsrc + "' does not exist")
212 if (options.branch is not None):
213 for branch in options.branch:
214 if (not os.path.isdir(os.path.join(compsrc, branch))):
215 parser.error("'" + os.path.join(compsrc, branch)
216 + "' does not exist")
218 # See if we can find the compilers
219 options.cc = find_executable(options.cc)
220 options.cxx = find_executable(options.cxx)
224 # Find a unique short name for the given set of paths. This searches
225 # back through path components until it finds unique component names
226 # among all given paths.
227 def get_path_abbrevs(paths):
228 # Find the number of common starting characters in the last component
230 unique_paths = list(paths)
232 class NotFoundException(Exception): pass
234 # Find a unique component of each path.
235 unique_bases = unique_paths[:]
237 while len(unique_paths) > 0:
238 bases = [os.path.basename(src) for src in unique_paths]
239 components = { c for c in bases }
240 # Account for single entry in paths.
241 if len(components) > 1 or len(components) == len(bases):
242 # We found something unique.
244 if bases.count(c) == 1:
245 index = bases.index(c)
246 unique_bases[index] = c
247 # Remove the corresponding path from the set under
249 unique_paths[index] = None
250 unique_paths = [ p for p in unique_paths if p is not None ]
251 unique_paths = [os.path.dirname(src) for src in unique_paths]
253 if len(unique_paths) > 0:
254 raise NotFoundException()
256 abbrevs = dict(zip(paths, [base for base in unique_bases]))
260 # Given a set of unique names, find a short character sequence that
261 # uniquely identifies them.
262 def get_short_abbrevs(unique_bases):
263 # Find a unique start character for each path base.
264 my_unique_bases = unique_bases[:]
265 unique_char_starts = unique_bases[:]
266 while len(my_unique_bases) > 0:
267 for start, char_tuple in enumerate(zip(*[base
268 for base in my_unique_bases])):
269 chars = { c for c in char_tuple }
270 # Account for single path.
271 if len(chars) > 1 or len(chars) == len(char_tuple):
272 # We found something unique.
274 if char_tuple.count(c) == 1:
275 index = char_tuple.index(c)
276 unique_char_starts[index] = start
277 # Remove the corresponding path from the set under
279 my_unique_bases[index] = None
280 my_unique_bases = [ b for b in my_unique_bases
284 if len(my_unique_bases) > 0:
285 raise NotFoundException()
287 abbrevs = [abbrev[start_index:start_index+3]
288 for abbrev, start_index
289 in zip([base for base in unique_bases],
290 [index for index in unique_char_starts])]
292 abbrevs = dict(zip(unique_bases, abbrevs))
296 class Builder(threading.Thread):
297 class ExecutableNotFound(Exception): pass
298 class FileNotExecutable(Exception): pass
300 def __init__(self, work_queue, jobs,
301 build_abbrev, source_abbrev, branch_abbrev,
304 self.work_queue = work_queue
307 self.cxx = options.cxx
308 self.build_abbrev = build_abbrev
309 self.source_abbrev = source_abbrev
310 self.branch_abbrev = branch_abbrev
311 self.build_prefix = options.builddir
312 self.install_prefix = options.prefix
313 self.options = options
314 self.component_abbrev = dict(
323 source, branch, build = self.work_queue.get()
324 self.dobuild(source, branch, build)
326 traceback.print_exc()
328 self.work_queue.task_done()
330 def execute(self, command, execdir, env, component):
331 prefix = self.component_abbrev[component.replace("-", "_")]
333 if not os.path.exists(execdir):
336 execenv = os.environ.copy()
338 for key, value in env.items():
341 self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
342 + " ".join(command));
345 proc = subprocess.Popen(command,
348 stdout=subprocess.PIPE,
349 stderr=subprocess.STDOUT)
351 line = proc.stdout.readline()
353 self.logger.info("[" + prefix + "] "
354 + str(line, "utf-8").rstrip())
355 line = proc.stdout.readline()
358 traceback.print_exc()
360 # Get a list of C++ include directories to pass to clang.
361 def get_includes(self):
362 # Assume we're building with g++ for now.
364 command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
366 self.logger.debug(command)
368 proc = subprocess.Popen(command,
369 stdout=subprocess.PIPE,
370 stderr=subprocess.STDOUT)
373 line = proc.stdout.readline()
375 self.logger.debug(line)
376 if re.search("End of search list", str(line)) is not None:
377 self.logger.debug("Stop Gather")
380 includes.append(str(line, "utf-8").strip())
381 if re.search("#include <...> search starts", str(line)) is not None:
382 self.logger.debug("Start Gather")
384 line = proc.stdout.readline()
386 traceback.print_exc()
387 self.logger.debug(includes)
390 def dobuild(self, source, branch, build):
393 ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
395 if branch is not None:
396 sbabbrev = get_short_abbrevs([ab for ab in self.branch_abbrev.values()])
398 prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + sbabbrev[self.branch_abbrev[branch]] + "-" + self.build_abbrev[build] + "]"
399 self.install_prefix += "/" + self.source_abbrev[source] + "/" + branch + "/" + build
400 build_suffix += self.source_abbrev[source] + "/" + branch + "/" + build
402 prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
403 self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
404 build_suffix += "/" + self.source_abbrev[source] + "/" + build
406 self.logger = logging.getLogger(prefix)
408 self.logger.debug(self.install_prefix)
410 # Assume we're building with gcc for now.
411 cxxincludes = self.get_includes()
412 cxxroot = cxxincludes[0]
413 cxxarch = os.path.basename(cxxincludes[1])
415 configure_flags = dict(
416 llvm=dict(debug=["--prefix=" + self.install_prefix,
417 "--with-extra-options=-Werror",
418 "--enable-assertions",
419 "--disable-optimized",
420 "--with-cxx-include-root=" + cxxroot,
421 "--with-cxx-include-arch=" + cxxarch],
422 release=["--prefix=" + self.install_prefix,
423 "--with-extra-options=-Werror",
424 "--enable-optimized",
425 "--with-cxx-include-root=" + cxxroot,
426 "--with-cxx-include-arch=" + cxxarch],
427 paranoid=["--prefix=" + self.install_prefix,
428 "--with-extra-options=-Werror",
429 "--enable-assertions",
430 "--enable-expensive-checks",
431 "--disable-optimized",
432 "--with-cxx-include-root=" + cxxroot,
433 "--with-cxx-include-arch=" + cxxarch]),
434 llvm_gcc=dict(debug=["--prefix=" + self.install_prefix,
436 "--program-prefix=llvm-",
437 "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
438 # Fortran install seems to be broken.
439 # "--enable-languages=c,c++,fortran"],
440 "--enable-languages=c,c++"],
441 release=["--prefix=" + self.install_prefix,
442 "--program-prefix=llvm-",
443 "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
444 # Fortran install seems to be broken.
445 # "--enable-languages=c,c++,fortran"],
446 "--enable-languages=c,c++"],
447 paranoid=["--prefix=" + self.install_prefix,
449 "--program-prefix=llvm-",
450 "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
451 # Fortran install seems to be broken.
452 # "--enable-languages=c,c++,fortran"]),
453 "--enable-languages=c,c++"]),
454 llvm2=dict(debug=["--prefix=" + self.install_prefix,
455 "--with-extra-options=-Werror",
456 "--enable-assertions",
457 "--disable-optimized",
458 "--with-llvmgccdir=" + self.install_prefix + "/bin",
459 "--with-cxx-include-root=" + cxxroot,
460 "--with-cxx-include-arch=" + cxxarch],
461 release=["--prefix=" + self.install_prefix,
462 "--with-extra-options=-Werror",
463 "--enable-optimized",
464 "--with-llvmgccdir=" + self.install_prefix + "/bin",
465 "--with-cxx-include-root=" + cxxroot,
466 "--with-cxx-include-arch=" + cxxarch],
467 paranoid=["--prefix=" + self.install_prefix,
468 "--with-extra-options=-Werror",
469 "--enable-assertions",
470 "--enable-expensive-checks",
471 "--disable-optimized",
472 "--with-llvmgccdir=" + self.install_prefix + "/bin",
473 "--with-cxx-include-root=" + cxxroot,
474 "--with-cxx-include-arch=" + cxxarch]),
475 gcc=dict(debug=["--prefix=" + self.install_prefix,
476 "--enable-checking"],
477 release=["--prefix=" + self.install_prefix],
478 paranoid=["--prefix=" + self.install_prefix,
479 "--enable-checking"]),
480 dragonegg=dict(debug=[],
484 configure_env = dict(
485 llvm=dict(debug=dict(CC=self.cc,
487 release=dict(CC=self.cc,
489 paranoid=dict(CC=self.cc,
491 llvm_gcc=dict(debug=dict(CC=self.cc,
493 release=dict(CC=self.cc,
495 paranoid=dict(CC=self.cc,
497 llvm2=dict(debug=dict(CC=self.cc,
499 release=dict(CC=self.cc,
501 paranoid=dict(CC=self.cc,
503 gcc=dict(debug=dict(CC=self.cc,
505 release=dict(CC=self.cc,
507 paranoid=dict(CC=self.cc,
509 dragonegg=dict(debug=dict(CC=self.cc,
511 release=dict(CC=self.cc,
513 paranoid=dict(CC=self.cc,
517 llvm=dict(debug=["-j" + str(self.jobs)],
518 release=["-j" + str(self.jobs)],
519 paranoid=["-j" + str(self.jobs)]),
520 llvm_gcc=dict(debug=["-j" + str(self.jobs),
522 release=["-j" + str(self.jobs),
524 paranoid=["-j" + str(self.jobs),
526 llvm2=dict(debug=["-j" + str(self.jobs)],
527 release=["-j" + str(self.jobs)],
528 paranoid=["-j" + str(self.jobs)]),
529 gcc=dict(debug=["-j" + str(self.jobs),
531 release=["-j" + str(self.jobs),
533 paranoid=["-j" + str(self.jobs),
535 dragonegg=dict(debug=["-j" + str(self.jobs)],
536 release=["-j" + str(self.jobs)],
537 paranoid=["-j" + str(self.jobs)]))
540 llvm=dict(debug=dict(),
543 llvm_gcc=dict(debug=dict(),
546 llvm2=dict(debug=dict(),
549 gcc=dict(debug=dict(),
552 dragonegg=dict(debug=dict(GCC=self.install_prefix + "/bin/gcc",
553 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
554 release=dict(GCC=self.install_prefix + "/bin/gcc",
555 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
556 paranoid=dict(GCC=self.install_prefix + "/bin/gcc",
557 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
559 make_install_flags = dict(
560 llvm=dict(debug=["install"],
562 paranoid=["install"]),
563 llvm_gcc=dict(debug=["install"],
565 paranoid=["install"]),
566 llvm2=dict(debug=["install"],
568 paranoid=["install"]),
569 gcc=dict(debug=["install"],
571 paranoid=["install"]),
572 dragonegg=dict(debug=["install"],
574 paranoid=["install"]))
576 make_install_env = dict(
577 llvm=dict(debug=dict(),
580 llvm_gcc=dict(debug=dict(),
583 llvm2=dict(debug=dict(),
586 gcc=dict(debug=dict(),
589 dragonegg=dict(debug=dict(),
593 make_check_flags = dict(
594 llvm=dict(debug=["check"],
597 llvm_gcc=dict(debug=["check"],
600 llvm2=dict(debug=["check"],
603 gcc=dict(debug=["check"],
606 dragonegg=dict(debug=["check"],
610 make_check_env = dict(
611 llvm=dict(debug=dict(),
614 llvm_gcc=dict(debug=dict(),
617 llvm2=dict(debug=dict(),
620 gcc=dict(debug=dict(),
623 dragonegg=dict(debug=dict(),
627 for component in components:
630 if (self.options.no_gcc):
631 if (comp == 'gcc' or comp == 'dragonegg' or comp == 'llvm2'):
632 self.logger.info("Skipping " + component + " in "
636 srcdir = source + "/" + comp.rstrip("2")
637 builddir = self.build_prefix + "/" + comp + "/" + build_suffix
638 installdir = self.install_prefix
640 if (branch is not None):
641 srcdir += "/" + branch
643 comp_key = comp.replace("-", "_")
645 config_args = configure_flags[comp_key][build][:]
646 config_args.extend(getattr(self.options,
647 "extra_" + comp_key.rstrip("2")
648 + "_config_flags").split())
650 self.logger.info("Configuring " + component + " in " + builddir)
651 self.configure(component, srcdir, builddir,
653 configure_env[comp_key][build])
655 self.logger.info("Building " + component + " in " + builddir)
656 self.logger.info("Build: make " + str(make_flags[comp_key][build]))
657 self.make(component, srcdir, builddir,
658 make_flags[comp_key][build],
659 make_env[comp_key][build])
661 if (not self.options.no_install):
662 self.logger.info("Installing " + component + " in " + installdir)
663 self.make(component, srcdir, builddir,
664 make_install_flags[comp_key][build],
665 make_install_env[comp_key][build])
667 self.logger.info("Testing " + component + " in " + builddir)
668 self.logger.info("Test: make "
669 + str(make_check_flags[comp_key][build]))
670 self.make(component, srcdir, builddir,
671 make_check_flags[comp_key][build],
672 make_check_env[comp_key][build])
675 def configure(self, component, srcdir, builddir, flags, env):
676 self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
679 configure_files = dict(
680 llvm=[(srcdir + "/configure", builddir + "/Makefile")],
681 llvm_gcc=[(srcdir + "/configure", builddir + "/Makefile"),
682 (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")],
683 llvm2=[(srcdir + "/configure", builddir + "/Makefile")],
684 gcc=[(srcdir + "/configure", builddir + "/Makefile"),
685 (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")],
690 for conf, mf in configure_files[component.replace("-", "_")]:
691 if not os.path.exists(conf):
693 if os.path.exists(conf) and os.path.exists(mf):
694 confstat = os.stat(conf)
695 makestat = os.stat(mf)
696 if confstat.st_mtime > makestat.st_mtime:
703 if not doconfig and not self.options.force_configure:
706 program = srcdir + "/configure"
707 if not is_executable(program):
711 args += ["--verbose"]
713 self.execute(args, builddir, env, component)
715 def make(self, component, srcdir, builddir, flags, env):
716 program = find_executable("make")
718 raise ExecutableNotFound
720 if not is_executable(program):
721 raise FileNotExecutable
725 self.execute(args, builddir, env, component)
728 build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
729 #components = ["llvm", "llvm-gcc", "llvm2", "gcc", "dragonegg"]
730 components = ["llvm", "llvm2", "gcc", "dragonegg"]
733 parser = optparse.OptionParser(version="%prog 1.0")
735 (options, args) = parser.parse_args()
736 check_options(parser, options, build_abbrev.keys());
739 logging.basicConfig(level=logging.DEBUG,
740 format='%(name)-13s: %(message)s')
742 logging.basicConfig(level=logging.INFO,
743 format='%(name)-13s: %(message)s')
745 source_abbrev = get_path_abbrevs(set(options.src))
748 if options.branch is not None:
749 branch_abbrev = get_path_abbrevs(set(options.branch))
751 work_queue = queue.Queue()
753 jobs = options.jobs // options.threads
757 numthreads = options.threads
759 logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
760 + str(numthreads) + " threads using " + str(jobs)
763 for t in range(numthreads):
764 builder = Builder(work_queue, jobs,
765 build_abbrev, source_abbrev, branch_abbrev,
767 builder.daemon = True
770 for build in set(options.build):
771 for source in set(options.src):
772 if options.branch is not None:
773 for branch in set(options.branch):
774 work_queue.put((source, branch, build))
776 work_queue.put((source, None, build))