X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=utils%2FNightlyTest.pl;h=d02e660ad1923ec5714cb6439d6706605aaaf13d;hb=75d1412cc2a2665cbc36850695f250fb09284d7c;hp=b8b6f4c56f6bdd6927e4ce08b1494bc6afe58365;hpb=30ac0c02855e3e9f6f94e3b3da0a895f77e4479b;p=oota-llvm.git diff --git a/utils/NightlyTest.pl b/utils/NightlyTest.pl index b8b6f4c56f6..d02e660ad19 100755 --- a/utils/NightlyTest.pl +++ b/utils/NightlyTest.pl @@ -1,4 +1,4 @@ -#!/usr/dcs/software/supported/bin/perl -w +#!/usr/bin/perl -w # # Program: NightlyTest.pl # @@ -7,12 +7,36 @@ # regressions and performance changes. This generates one web page a # day which can be used to access this information. # -# Syntax: NightlyTest.pl +# Syntax: NightlyTest.pl [OPTIONS] [CVSROOT BUILDDIR WEBDIR] +# where +# OPTIONS may include one or more of the following: +# -nocheckout Do not create, checkout, update, or configure +# the source tree. +# -noremove Do not remove the BUILDDIR after it has been built. +# -notest Do not even attempt to run the test programs. Implies +# -norunningtests. +# -norunningtests Do not run the Olden benchmark suite with +# LARGE_PROBLEM_SIZE enabled. +# -parallel Run two parallel jobs with GNU Make. +# -enable-linscan Enable linearscan tests +# +# CVSROOT is the CVS repository from which the tree will be checked out, +# specified either in the full :method:user@host:/dir syntax, or +# just /dir if using a local repo. +# BUILDDIR is the directory where sources for this test run will be checked out +# AND objects for this test run will be built. This directory MUST NOT +# exist before the script is run; it will be created by the cvs checkout +# process and erased (unless -noremove is specified; see above.) +# WEBDIR is the directory into which the test results web page will be written, +# AND in which the "index.html" is assumed to be a symlink to the most recent +# copy of the results. This directory MUST exist before the script is run. # use POSIX qw(strftime); -my $HOME = $ENV{HOME}; -my $CVSRootDir = "/home/vadve/vadve/Research/DynOpt/CVSRepository"; +my $HOME = $ENV{'HOME'}; +my $CVSRootDir = $ENV{'CVSROOT'}; + $CVSRootDir = "/home/vadve/shared/PublicCVS" + unless $CVSRootDir; my $BuildDir = "$HOME/buildtest"; my $WebDir = "$HOME/cvs/testresults-X86"; @@ -22,6 +46,7 @@ my $DATE = sprintf "%4d-%02d-%02d", $TIME[5]+1900, $TIME[4]+1, $TIME[3]; my $DateString = strftime "%B %d, %Y", localtime; sub ReadFile { + undef $/; if (open (FILE, $_[0])) { my $Ret = ; close FILE; @@ -40,7 +65,10 @@ sub WriteFile { # (filename, contents) sub GetRegex { # (Regex with ()'s, value) $_[1] =~ /$_[0]/m; - return $1; + if (defined($1)) { + return $1; + } + return "?"; } sub AddPreTag { # Add pre tags around nonempty list, or convert to "none" @@ -75,12 +103,23 @@ sub DiffFiles { return ($Added, $Removed); } +# FormatTime - Convert a time from 1m23.45 into 83.45 +sub FormatTime { + my $Time = shift; + if ($Time =~ m/([0-9]+)m([0-9.]+)/) { + $Time = sprintf("%7.4f", $1*60.0+$2); + } + return $Time; +} + # Command line argument settings... my $NOCHECKOUT = 0; my $NOREMOVE = 0; my $NOTEST = 0; +my $NORUNNINGTESTS = 0; my $MAKEOPTS = ""; +my $ENABLELINEARSCAN = ""; # Parse arguments... while (scalar(@ARGV) and ($_ = $ARGV[0], /^[-+]/)) { @@ -88,25 +127,25 @@ while (scalar(@ARGV) and ($_ = $ARGV[0], /^[-+]/)) { last if /^--$/; # Stop processing arguments on -- # List command line options here... - if (/^-nocheckout$/) { $NOCHECKOUT = 1; next; } - if (/^-noremove$/) { $NOREMOVE = 1; next; } - if (/^-notest$/) { $NOTEST = 1; next; } - if (/^-parallel$/) { $MAKEOPTS = "-j2 -l3.0"; next; } + if (/^-nocheckout$/) { $NOCHECKOUT = 1; next; } + if (/^-noremove$/) { $NOREMOVE = 1; next; } + if (/^-notest$/) { $NOTEST = 1; $NORUNNINGTESTS = 1; next; } + if (/^-norunningtests$/) { $NORUNNINGTESTS = 1; next; } + if (/^-parallel$/) { $MAKEOPTS = "-j2 -l3.0"; next; } + if (/^-enable-linscan$/) { $ENABLELINEARSCAN = "ENABLE_LINEARSCAN=1"; next; } print "Unknown option: $_ : ignoring!\n"; } die "Must specify 0 or 3 options!" if (@ARGV != 0 and @ARGV != 3); -# FIXME: This should just be utils/... -my $Template = "$HOME/llvm/utils/NightlyTestTemplate.html"; - if (@ARGV == 3) { $CVSRootDir = $ARGV[0]; $BuildDir = $ARGV[1]; $WebDir = $ARGV[2]; } +my $Template = "$BuildDir/llvm/utils/NightlyTestTemplate.html"; my $Prefix = "$WebDir/$DATE"; if (0) { @@ -121,15 +160,17 @@ if (0) { # Create the CVS repository directory # if (!$NOCHECKOUT) { - mkdir $BuildDir or die "Could not create CVS checkout directory!"; + mkdir $BuildDir or die "Could not create CVS checkout directory $BuildDir!"; } -chdir $BuildDir or die "Could not change to CVS checkout directory!"; +chdir $BuildDir or die "Could not change to CVS checkout directory $BuildDir!"; # # Check out the llvm tree, saving CVS messages to the cvs log... # -system "(time -p cvs -d $CVSRootDir co llvm) > $Prefix-CVS-Log.txt 2>&1" +$CVSOPT = ""; +$CVSOPT = "-z3" if $CVSRootDir =~ /^:ext:/; # Use compression if going over ssh. +system "(time -p cvs $CVSOPT -d $CVSRootDir co llvm) > $Prefix-CVS-Log.txt 2>&1" if (!$NOCHECKOUT); chdir "llvm" or die "Could not change into llvm directory!"; @@ -137,7 +178,6 @@ chdir "llvm" or die "Could not change into llvm directory!"; system "cvs up -P -d > /dev/null 2>&1" if (!$NOCHECKOUT); # Read in the HTML template file... -undef $/; my $TemplateContents = ReadFile $Template; @@ -145,28 +185,27 @@ my $TemplateContents = ReadFile $Template; # Get some static statistics about the current state of CVS # my $CVSCheckoutTime = GetRegex "([0-9.]+)", `grep '^real' $Prefix-CVS-Log.txt`; -my $NumFilesInCVS = `grep ^U $Prefix-CVS-Log.txt | wc -l` + 0; -my $NumDirsInCVS = `grep '^cvs checkout' $Prefix-CVS-Log.txt | wc -l` + 0; +my $NumFilesInCVS = `egrep '^U' $Prefix-CVS-Log.txt | wc -l` + 0; +my $NumDirsInCVS = `egrep '^cvs (checkout|server|update):' $Prefix-CVS-Log.txt | wc -l` + 0; $LOC = GetRegex "([0-9]+) +total", `wc -l \`utils/getsrcs.sh\` | grep total`; # # Build the entire tree, saving build messages to the build log # if (!$NOCHECKOUT) { - # Change the Makefile.config to build into the local directory... - rename "Makefile.config", "Makefile.config.orig"; - system "sed '/^LLVM_OBJ_DIR/d' < Makefile.config.orig > Makefile.config"; - system "echo >> Makefile.config"; - system "echo 'LLVM_OBJ_DIR := .' >> Makefile.config"; - - # Change the Makefile.config to not strip executables... - system "echo 'KEEP_SYMBOLS := 1' >> Makefile.config"; + system "(time -p ./configure --enable-jit --enable-spec --with-objroot=.) > $Prefix-Build-Log.txt 2>&1"; # Build the entire tree, capturing the output into $Prefix-Build-Log.txt - system "(time -p gmake $MAKEOPTS) > $Prefix-Build-Log.txt 2>&1"; + system "(time -p gmake $MAKEOPTS) >> $Prefix-Build-Log.txt 2>&1"; } +sub GetRegexNum { + my ($Regex, $Num, $Regex2, $File) = @_; + my @Items = split "\n", `grep '$Regex' $File`; + return GetRegex $Regex2, $Items[$Num]; +} + # # Get some statistics about the build... # @@ -174,28 +213,36 @@ my @Linked = split '\n', `grep Linking $Prefix-Build-Log.txt`; my $NumExecutables = scalar(grep(/executable/, @Linked)); my $NumLibraries = scalar(grep(!/executable/, @Linked)); my $NumObjects = `grep '^Compiling' $Prefix-Build-Log.txt | wc -l` + 0; -my $BuildTimeU = GetRegex "([0-9.]+)", `grep '^user' $Prefix-Build-Log.txt`; -my $BuildTimeS = GetRegex "([0-9.]+)", `grep '^sys' $Prefix-Build-Log.txt`; -my $BuildWallTime = GetRegex "([0-9.]+)", `grep '^real' $Prefix-Build-Log.txt`; + +my $ConfigTimeU = GetRegexNum "^user", 0, "([0-9.]+)", "$Prefix-Build-Log.txt"; +my $ConfigTimeS = GetRegexNum "^sys", 0, "([0-9.]+)", "$Prefix-Build-Log.txt"; +my $ConfigTime = $ConfigTimeU+$ConfigTimeS; # ConfigTime = User+System +my $ConfigWallTime = GetRegexNum "^real", 0,"([0-9.]+)","$Prefix-Build-Log.txt"; + +my $BuildTimeU = GetRegexNum "^user", 1, "([0-9.]+)", "$Prefix-Build-Log.txt"; +my $BuildTimeS = GetRegexNum "^sys", 1, "([0-9.]+)", "$Prefix-Build-Log.txt"; my $BuildTime = $BuildTimeU+$BuildTimeS; # BuildTime = User+System +my $BuildWallTime = GetRegexNum "^real", 1, "([0-9.]+)","$Prefix-Build-Log.txt"; + my $BuildError = ""; -if (`grep '^gmake: .*Error' $Prefix-Build-Log.txt | wc -l` + 0) { - $BuildError = "

Build error: compilation " - . "aborted

"; +if (`grep '^gmake[^:]*: .*Error' $Prefix-Build-Log.txt | wc -l` + 0 || + `grep '^gmake: \*\*\*.*Stop.' $Prefix-Build-Log.txt | wc -l`+0) { + $BuildError = "

Build error: compilation " . + "aborted

"; + print "BUILD ERROR\n"; } - # # Get warnings from the build # -my @Warn = split "\n", `grep -E 'warning:|Entering dir' $Prefix-Build-Log.txt`; +my @Warn = split "\n", `egrep 'warning:|Entering dir' $Prefix-Build-Log.txt`; my @Warnings; my $CurDir = ""; foreach $Warning (@Warn) { if ($Warning =~ m/Entering directory \`([^\`]+)\'/) { $CurDir = $1; # Keep track of directory warning is in... - if ($CurDir =~ m|$BuildDir/llvm/(.*)|) { # Remove buildir prefix if included + if ($CurDir =~ m#$BuildDir/llvm/(.*)#) { # Remove buildir prefix if included $CurDir = $1; } } else { @@ -209,10 +256,14 @@ $WarningsFile =~ s/:[0-9]+:/::/g; # Emit the warnings file, so we can diff... WriteFile "$WebDir/$DATE-Warnings.txt", $WarningsFile . "\n"; my ($WarningsAdded, $WarningsRemoved) = DiffFiles "-Warnings.txt"; + +# Output something to stdout if something has changed +print "ADDED WARNINGS:\n$WarningsAdded\n\n" if (length $WarningsAdded); +print "REMOVED WARNINGS:\n$WarningsRemoved\n\n" if (length $WarningsRemoved); + $WarningsAdded = AddPreTag $WarningsAdded; $WarningsRemoved = AddPreTag $WarningsRemoved; - # # Get some statistics about CVS commits over the current day... # @@ -221,19 +272,18 @@ $WarningsRemoved = AddPreTag $WarningsRemoved; # Extract some information from the CVS history... use a hash so no duplicate # stuff is stored. -my (%AddedFiles, %ModifiedFiles, %RemovedFiles, - %UsersCommitted, %UsersUpdated); +my (%AddedFiles, %ModifiedFiles, %RemovedFiles, %UsersCommitted, %UsersUpdated); my $DateRE = "[-:0-9 ]+\\+[0-9]+"; # Loop over every record from the CVS history, filling in the hashes. foreach $File (@CVSHistory) { my ($Type, $Date, $UID, $Rev, $Filename); - if ($File =~ /([AMRUGC])\s($DateRE)\s([^\s]+)\s+([0-9.]+)\s+([^\s]+)\s+([^\s]+)/) { + if ($File =~ /([AMRUGC]) ($DateRE) ([^ ]+) +([^ ]+) +([^ ]+) +([^ ]+)/) { ($Type, $Date, $UID, $Rev, $Filename) = ($1, $2, $3, $4, "$6/$5"); - } elsif ($File =~ /([W])\s($DateRE)\s([^\s]+) +([^\s]+)\s+([^\s]+)/) { - ($Type, $Date, $UID, $Rev, $Filename) = ($1, $2, $3, $4, "$6/$5"); - } elsif ($File =~ /([O]) ($DateRE) ([^ ]+) +([^\s]+)/) { + } elsif ($File =~ /([W]) ($DateRE) ([^ ]+)/) { + ($Type, $Date, $UID, $Rev, $Filename) = ($1, $2, $3, "", ""); + } elsif ($File =~ /([O]) ($DateRE) ([^ ]+) +([^ ]+)/) { ($Type, $Date, $UID, $Rev, $Filename) = ($1, $2, $3, "", "$4/"); } else { print "UNMATCHABLE: $File\n"; @@ -258,28 +308,40 @@ foreach $File (@CVSHistory) { my $UserCommitList = join "\n", keys %UsersCommitted; my $UserUpdateList = join "\n", keys %UsersUpdated; -my $AddedFilesList = AddPreTag join "\n", keys %AddedFiles; -my $ModifiedFilesList = AddPreTag join "\n", keys %ModifiedFiles; -my $RemovedFilesList = AddPreTag join "\n", keys %RemovedFiles; +my $AddedFilesList = AddPreTag join "\n", sort keys %AddedFiles; +my $ModifiedFilesList = AddPreTag join "\n", sort keys %ModifiedFiles; +my $RemovedFilesList = AddPreTag join "\n", sort keys %RemovedFiles; my $TestError = 1; -my $ProgramsTable; +my $SingleSourceProgramsTable; +my $MultiSourceProgramsTable; +my $ExternalProgramsTable; -# If we build the tree successfully, the nightly programs tests... -if ($BuildError eq "") { - chdir "test/Programs" or die "Could not change into programs testdir!"; + +sub TestDirectory { + my $SubDir = shift; + + chdir "test/Programs/$SubDir" or + die "Could not change into test/Programs/$SubDir testdir!"; # Run the programs tests... creating a report.nightly.html file if (!$NOTEST) { - system "gmake $MAKEOPTS report.nightly.html TEST=nightly " - . "RUNTIMELIMIT=300 > $Prefix-ProgramTest.txt 2>&1"; + system "gmake -k $MAKEOPTS $ENABLELINEARSCAN report.nightly.html " + . "TEST=nightly > $Prefix-$SubDir-ProgramTest.txt 2>&1"; } else { - system "gunzip $Prefix-ProgramTest.txt.gz"; + system "gunzip $Prefix-$SubDir-ProgramTest.txt.gz"; } - if (`grep '^gmake: .*Error' $Prefix-ProgramTest.txt | wc -l` + 0) { + my $ProgramsTable; + if (`grep '^gmake[^:]: .*Error' $Prefix-$SubDir-ProgramTest.txt | wc -l` + 0){ + $TestError = 1; + $ProgramsTable = "

Error running tests!

"; + print "ERROR TESTING\n"; + } elsif (`grep '^gmake[^:]: .*No rule to make target' $Prefix-$SubDir-ProgramTest.txt | wc -l` + 0) { $TestError = 1; - $ProgramsTable = "

Error running tests!

"; + $ProgramsTable = + "

Makefile error running tests!

"; + print "ERROR TESTING\n"; } else { $TestError = 0; $ProgramsTable = ReadFile "report.nightly.html"; @@ -287,12 +349,23 @@ if ($BuildError eq "") { # # Create a list of the tests which were run... # - system "grep -E 'TEST-(PASS|FAIL)' < $Prefix-ProgramTest.txt " - . "| sort > $Prefix-Tests.txt"; + system "egrep 'TEST-(PASS|FAIL)' < $Prefix-$SubDir-ProgramTest.txt " + . "| sort > $Prefix-$SubDir-Tests.txt"; } # Compress the test output - system "gzip -f $Prefix-ProgramTest.txt"; + system "gzip -f $Prefix-$SubDir-ProgramTest.txt"; + chdir "../../.." or die "Cannot return to parent directory!"; + return $ProgramsTable; +} + +# If we build the tree successfully, run the nightly programs tests... +if ($BuildError eq "") { + $SingleSourceProgramsTable = TestDirectory("SingleSource"); + $MultiSourceProgramsTable = TestDirectory("MultiSource"); + $ExternalProgramsTable = TestDirectory("External"); + system "cat $Prefix-SingleSource-Tests.txt $Prefix-MultiSource-Tests.txt ". + " $Prefix-External-Tests.txt | sort > $Prefix-Tests.txt"; } my ($TestsAdded, $TestsRemoved, $TestsFixed, $TestsBroken) = ("","","",""); @@ -328,12 +401,79 @@ if ($TestError) { $TestsRemoved = "$TestsRemoved$Test\n" if (!exists $NewTests{$Test}); } + print "\nTESTS ADDED: \n\n$TestsAdded\n\n" if (length $TestsAdded); + print "\nTESTS REMOVED:\n\n$TestsRemoved\n\n" if (length $TestsRemoved); + print "\nTESTS FIXED: \n\n$TestsFixed\n\n" if (length $TestsFixed); + print "\nTESTS BROKEN: \n\n$TestsBroken\n\n" if (length $TestsBroken); + $TestsAdded = AddPreTag $TestsAdded; $TestsRemoved = AddPreTag $TestsRemoved; $TestsFixed = AddPreTag $TestsFixed; $TestsBroken = AddPreTag $TestsBroken; } + +# If we built the tree successfully, runs of the Olden suite with +# LARGE_PROBLEM_SIZE on so that we can get some "running" statistics. +if ($BuildError eq "") { + my ($NATTime, $CBETime, $LLCTime, $JITTime, $OptTime, $BytecodeSize, + $MachCodeSize) = ("","","","","","",""); + if (!$NORUNNINGTESTS) { + chdir "test/Programs/MultiSource/Benchmarks/Olden" or die "Olden tests moved?"; + + # Clean out previous results... + system "gmake $MAKEOPTS clean > /dev/null 2>&1"; + + # Run the nightly test in this directory, with LARGE_PROBLEM_SIZE enabled! + system "gmake -k $MAKEOPTS report.nightly.raw.out TEST=nightly " . + " LARGE_PROBLEM_SIZE=1 > /dev/null 2>&1"; + system "cp report.nightly.raw.out $Prefix-Olden-tests.txt"; + } else { + system "gunzip $Prefix-Olden-tests.txt.gz"; + } + + # Now we know we have $Prefix-Olden-tests.txt as the raw output file. Split + # it up into records and read the useful information. + my @Records = split />>> ========= /, ReadFile "$Prefix-Olden-tests.txt"; + shift @Records; # Delete the first (garbage) record + + # Loop over all of the records, summarizing them into rows for the running + # totals file. + my $WallTimeRE = "[A-Za-z0-9.: ]+\\(([0-9.]+) wall clock"; + foreach $Rec (@Records) { + my $rNATTime = GetRegex 'TEST-RESULT-nat-time: real\s*([.0-9m]+)', $Rec; + my $rCBETime = GetRegex 'TEST-RESULT-cbe-time: real\s*([.0-9m]+)', $Rec; + my $rLLCTime = GetRegex 'TEST-RESULT-llc-time: real\s*([.0-9m]+)', $Rec; + my $rJITTime = GetRegex 'TEST-RESULT-jit-time: real\s*([.0-9m]+)', $Rec; + my $rOptTime = GetRegex "TEST-RESULT-compile: $WallTimeRE", $Rec; + my $rBytecodeSize = GetRegex 'TEST-RESULT-compile: *([0-9]+)', $Rec; + my $rMachCodeSize = GetRegex 'TEST-RESULT-jit-machcode: *([0-9]+).*bytes of machine code', $Rec; + + $NATTime .= " " . FormatTime($rNATTime); + $CBETime .= " " . FormatTime($rCBETime); + $LLCTime .= " " . FormatTime($rLLCTime); + $JITTime .= " " . FormatTime($rJITTime); + $OptTime .= " $rOptTime"; + $BytecodeSize .= " $rBytecodeSize"; + $MachCodeSize .= " $rMachCodeSize"; + } + + # Now that we have all of the numbers we want, add them to the running totals + # files. + AddRecord($NATTime, "running_Olden_nat_time.txt"); + AddRecord($CBETime, "running_Olden_cbe_time.txt"); + AddRecord($LLCTime, "running_Olden_llc_time.txt"); + AddRecord($JITTime, "running_Olden_jit_time.txt"); + AddRecord($OptTime, "running_Olden_opt_time.txt"); + AddRecord($BytecodeSize, "running_Olden_bytecode.txt"); + AddRecord($MachCodeSize, "running_Olden_machcode.txt"); + + system "gzip -f $Prefix-Olden-tests.txt"; +} + + + + # # Get a list of the previous days that we can link to... # @@ -344,14 +484,28 @@ splice @PrevDays, 20; # Trim down list to something reasonable... my $PrevDaysList = # Format list for sidebar join "\n ", map { "$_
" } @PrevDays; +# +# Start outputing files into the web directory +# +chdir $WebDir or die "Could not change into web directory!"; + +# Add information to the files which accumulate information for graphs... +AddRecord($LOC, "running_loc.txt"); +AddRecord($BuildTime, "running_build_time.txt"); + +# +# Rebuild the graphs now... +# +$GNUPLOT = "/usr/dcs/software/supported/bin/gnuplot"; +$GNUPLOT = "gnuplot" if ! -x $GNUPLOT; +$PlotScriptFilename = "$BuildDir/llvm/utils/NightlyTest.gnuplot"; +system ($GNUPLOT, $PlotScriptFilename); # # Remove the cvs tree... # -chdir $WebDir or die "Could not change into web directory!"; system "rm -rf $BuildDir" if (!$NOCHECKOUT and !$NOREMOVE); - # # Print out information... # @@ -365,7 +519,6 @@ if (0) { print "WARNINGS:\n $WarningsList\n"; - print "Users committed: $UserCommitList\n"; print "Added Files: \n $AddedFilesList\n"; print "Modified Files: \n $ModifiedFilesList\n"; @@ -390,15 +543,11 @@ system "ln -sf $DATE.html index.html"; sub AddRecord { my ($Val, $Filename) = @_; my @Records; - if (open FILE, $Filename) { + if (open FILE, "$WebDir/$Filename") { @Records = grep !/$DATE/, split "\n", ; close FILE; } push @Records, "$DATE: $Val"; - WriteFile $Filename, (join "\n", @Records) . "\n"; + WriteFile "$WebDir/$Filename", (join "\n", @Records) . "\n"; return @Records; } - -# Add information to the files which accumulate information for graphs... -AddRecord($LOC, "running_loc.txt"); -AddRecord($BuildTime, "running_build_time.txt");