From f20581a4b094f9d773c19da307b2f1f45757f84a Mon Sep 17 00:00:00 2001 From: Tom Jackson Date: Wed, 4 Oct 2017 10:31:05 -0700 Subject: [PATCH] Benchmark comparison Summary: I couldn't find anything that consumes `--json` output, so I made this. It prints out benchmark results from two runs, comparing each benchmark to its corresponding baseline. This should help with benchmarking changes across revisions. Along the way: - Two small transparent `struct`s replaces the use of `tuple`s everywhere. - New output JSON format (additive), formatted as array of arrays so order is preserved. - New comparison binary in `folly/tools/benchmark_compare` Sample output: {P58307694} Reviewed By: yfeldblum Differential Revision: D5908017 fbshipit-source-id: d7411e22b459db16bd897f656e48ea4e896cb1bf --- folly/Benchmark.cpp | 167 +++++++++++++++++++++++++------ folly/Benchmark.h | 27 +++++ folly/tools/BenchmarkCompare.cpp | 45 +++++++++ 3 files changed, 209 insertions(+), 30 deletions(-) create mode 100644 folly/tools/BenchmarkCompare.cpp diff --git a/folly/Benchmark.cpp b/folly/Benchmark.cpp index 8130646d..83d162a1 100644 --- a/folly/Benchmark.cpp +++ b/folly/Benchmark.cpp @@ -23,12 +23,14 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -36,6 +38,7 @@ using namespace std; DEFINE_bool(benchmark, false, "Run benchmarks."); DEFINE_bool(json, false, "Output in JSON format."); +DEFINE_bool(json_verbose, false, "Output in verbose JSON format."); DEFINE_string( bm_regex, @@ -68,9 +71,8 @@ std::chrono::high_resolution_clock::duration BenchmarkSuspender::timeSpent; typedef function BenchmarkFun; - -vector>& benchmarks() { - static vector> _benchmarks; +vector& benchmarks() { + static vector _benchmarks; return _benchmarks; } @@ -89,12 +91,11 @@ BENCHMARK(FB_FOLLY_GLOBAL_BENCHMARK_BASELINE) { size_t getGlobalBenchmarkBaselineIndex() { const char *global = FB_STRINGIZE_X2(FB_FOLLY_GLOBAL_BENCHMARK_BASELINE); auto it = std::find_if( - benchmarks().begin(), - benchmarks().end(), - [global](const tuple &v) { - return get<1>(v) == global; - } - ); + benchmarks().begin(), + benchmarks().end(), + [global](const detail::BenchmarkRegistration& v) { + return v.name == global; + }); CHECK(it != benchmarks().end()); return size_t(std::distance(benchmarks().begin(), it)); } @@ -104,7 +105,7 @@ size_t getGlobalBenchmarkBaselineIndex() { void detail::addBenchmarkImpl(const char* file, const char* name, BenchmarkFun fun) { - benchmarks().emplace_back(file, name, std::move(fun)); + benchmarks().push_back({file, name, std::move(fun)}); } /** @@ -243,14 +244,14 @@ static string metricReadable(double n, unsigned int decimals) { } static void printBenchmarkResultsAsTable( - const vector >& data) { + const vector& data) { // Width available static const unsigned int columns = 76; // Compute the longest benchmark name size_t longestName = 0; - FOR_EACH_RANGE (i, 1, benchmarks().size()) { - longestName = max(longestName, get<1>(benchmarks()[i]).size()); + for (auto& bm : benchmarks()) { + longestName = max(longestName, bm.name.size()); } // Print a horizontal rule @@ -270,14 +271,14 @@ static void printBenchmarkResultsAsTable( string lastFile; for (auto& datum : data) { - auto file = get<0>(datum); + auto file = datum.file; if (file != lastFile) { // New file starting header(file); lastFile = file; } - string s = get<1>(datum); + string s = datum.name; if (s == "-") { separator('-'); continue; @@ -287,11 +288,11 @@ static void printBenchmarkResultsAsTable( s.erase(0, 1); useBaseline = true; } else { - baselineNsPerIter = get<2>(datum); + baselineNsPerIter = datum.timeInNs; useBaseline = false; } s.resize(columns - 29, ' '); - auto nsPerIter = get<2>(datum); + auto nsPerIter = datum.timeInNs; auto secPerIter = nsPerIter / 1E9; auto itersPerSec = (secPerIter == 0) ? std::numeric_limits::infinity() @@ -316,29 +317,136 @@ static void printBenchmarkResultsAsTable( } static void printBenchmarkResultsAsJson( - const vector >& data) { + const vector& data) { dynamic d = dynamic::object; for (auto& datum: data) { - d[std::get<1>(datum)] = std::get<2>(datum) * 1000.; + d[datum.name] = datum.timeInNs * 1000.; } printf("%s\n", toPrettyJson(d).c_str()); } -static void printBenchmarkResults( - const vector >& data) { +static void printBenchmarkResultsAsVerboseJson( + const vector& data) { + dynamic d; + benchmarkResultsToDynamic(data, d); + printf("%s\n", toPrettyJson(d).c_str()); +} - if (FLAGS_json) { +static void printBenchmarkResults(const vector& data) { + if (FLAGS_json_verbose) { + printBenchmarkResultsAsVerboseJson(data); + } else if (FLAGS_json) { printBenchmarkResultsAsJson(data); } else { printBenchmarkResultsAsTable(data); } } +void benchmarkResultsToDynamic( + const vector& data, + dynamic& out) { + out = dynamic::array; + for (auto& datum : data) { + out.push_back(dynamic::array(datum.file, datum.name, datum.timeInNs)); + } +} + +void benchmarkResultsFromDynamic( + const dynamic& d, + vector& results) { + for (auto& datum : d) { + results.push_back( + {datum[0].asString(), datum[1].asString(), datum[2].asDouble()}); + } +} + +static pair resultKey( + const detail::BenchmarkResult& result) { + return pair(result.file, result.name); +} + +void printResultComparison( + const vector& base, + const vector& test) { + map, double> baselines; + + for (auto& baseResult : base) { + baselines[resultKey(baseResult)] = baseResult.timeInNs; + } + // + // Width available + static const unsigned int columns = 76; + + // Compute the longest benchmark name + size_t longestName = 0; + for (auto& datum : test) { + longestName = max(longestName, datum.name.size()); + } + + // Print a horizontal rule + auto separator = [&](char pad) { puts(string(columns, pad).c_str()); }; + + // Print header for a file + auto header = [&](const string& file) { + separator('='); + printf("%-*srelative time/iter iters/s\n", columns - 28, file.c_str()); + separator('='); + }; + + string lastFile; + + for (auto& datum : test) { + folly::Optional baseline = + folly::get_optional(baselines, resultKey(datum)); + auto file = datum.file; + if (file != lastFile) { + // New file starting + header(file); + lastFile = file; + } + + string s = datum.name; + if (s == "-") { + separator('-'); + continue; + } + if (s[0] == '%') { + s.erase(0, 1); + } + s.resize(columns - 29, ' '); + auto nsPerIter = datum.timeInNs; + auto secPerIter = nsPerIter / 1E9; + auto itersPerSec = (secPerIter == 0) + ? std::numeric_limits::infinity() + : (1 / secPerIter); + if (!baseline) { + // Print without baseline + printf( + "%*s %9s %7s\n", + static_cast(s.size()), + s.c_str(), + readableTime(secPerIter, 2).c_str(), + metricReadable(itersPerSec, 2).c_str()); + } else { + // Print with baseline + auto rel = *baseline / nsPerIter * 100.0; + printf( + "%*s %7.2f%% %9s %7s\n", + static_cast(s.size()), + s.c_str(), + rel, + readableTime(secPerIter, 2).c_str(), + metricReadable(itersPerSec, 2).c_str()); + } + } + separator('='); +} + void runBenchmarks() { CHECK(!benchmarks().empty()); - vector> results; + vector results; results.reserve(benchmarks().size() - 1); std::unique_ptr bmRegex; @@ -351,21 +459,20 @@ void runBenchmarks() { size_t baselineIndex = getGlobalBenchmarkBaselineIndex(); auto const globalBaseline = - runBenchmarkGetNSPerIteration(get<2>(benchmarks()[baselineIndex]), 0); + runBenchmarkGetNSPerIteration(benchmarks()[baselineIndex].func, 0); FOR_EACH_RANGE (i, 0, benchmarks().size()) { if (i == baselineIndex) { continue; } double elapsed = 0.0; - if (get<1>(benchmarks()[i]) != "-") { // skip separators - if (bmRegex && !boost::regex_search(get<1>(benchmarks()[i]), *bmRegex)) { + auto& bm = benchmarks()[i]; + if (bm.name != "-") { // skip separators + if (bmRegex && !boost::regex_search(bm.name, *bmRegex)) { continue; } - elapsed = runBenchmarkGetNSPerIteration(get<2>(benchmarks()[i]), - globalBaseline); + elapsed = runBenchmarkGetNSPerIteration(bm.func, globalBaseline); } - results.emplace_back(get<0>(benchmarks()[i]), - get<1>(benchmarks()[i]), elapsed); + results.push_back({bm.file, bm.name, elapsed}); } // PLEASE MAKE NOISE. MEASUREMENTS DONE. diff --git a/folly/Benchmark.h b/folly/Benchmark.h index 01e8ffee..dbe12145 100644 --- a/folly/Benchmark.h +++ b/folly/Benchmark.h @@ -55,6 +55,19 @@ namespace detail { using TimeIterPair = std::pair; +using BenchmarkFun = std::function; + +struct BenchmarkRegistration { + std::string file; + std::string name; + BenchmarkFun func; +}; + +struct BenchmarkResult { + std::string file; + std::string name; + double timeInNs; +}; /** * Adds a benchmark wrapped in a std::function. Only used @@ -280,6 +293,20 @@ auto makeUnpredictable(T& datum) -> typename std::enable_if< #endif +struct dynamic; + +void benchmarkResultsToDynamic( + const std::vector& data, + dynamic&); + +void benchmarkResultsFromDynamic( + const dynamic&, + std::vector&); + +void printResultComparison( + const std::vector& base, + const std::vector& test); + } // namespace folly /** diff --git a/folly/tools/BenchmarkCompare.cpp b/folly/tools/BenchmarkCompare.cpp new file mode 100644 index 00000000..3eae93b0 --- /dev/null +++ b/folly/tools/BenchmarkCompare.cpp @@ -0,0 +1,45 @@ +/* + * Copyright 2017-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +using namespace std; + +namespace folly { + +vector resultsFromFile(const std::string& filename) { + string content; + readFile(filename.c_str(), content); + vector ret; + benchmarkResultsFromDynamic(parseJson(content), ret); + return ret; +} + +void compareBenchmarkResults(const std::string& base, const std::string& test) { + printResultComparison(resultsFromFile(base), resultsFromFile(test)); +} + +} // namespace folly + +int main(int argc, char** argv) { + folly::init(&argc, &argv); + CHECK_GT(argc, 2); + folly::compareBenchmarkResults(argv[1], argv[2]); + return 0; +} -- 2.34.1