From 05bf65cb10758d1f2143ef41123cef7db057c885 Mon Sep 17 00:00:00 2001 From: weiyu Date: Wed, 16 Dec 2020 00:51:31 -0800 Subject: [PATCH] update scripts --- ...ssertion_test.sh => app_assertion_test.sh} | 27 +++++++------------ scripts/{test_all.sh => app_test_all.sh} | 14 +++++----- scripts/calculator.py | 12 +++++++++ scripts/do_test_all.sh | 20 ++++++++++++++ scripts/run.sh | 19 +++++-------- scripts/setup.sh | 8 ++++-- 6 files changed, 62 insertions(+), 38 deletions(-) rename scripts/{assertion_test.sh => app_assertion_test.sh} (69%) mode change 100644 => 100755 rename scripts/{test_all.sh => app_test_all.sh} (62%) create mode 100755 scripts/do_test_all.sh diff --git a/scripts/assertion_test.sh b/scripts/app_assertion_test.sh old mode 100644 new mode 100755 similarity index 69% rename from scripts/assertion_test.sh rename to scripts/app_assertion_test.sh index 8ee5011..3d59e27 --- a/scripts/assertion_test.sh +++ b/scripts/app_assertion_test.sh @@ -3,7 +3,7 @@ MABAINLIB="../src" MABAINDIR="mabain/examples" -TESTS="silo" +TESTS="silo mabain" TOTAL_RUN=$1 @@ -12,7 +12,6 @@ if [ -z "$1" ]; then fi function run_silo_test { - echo "Silo assertion test" COUNT_ASSERT=0 EXE='./dbtest --verbose -t 5' @@ -29,13 +28,12 @@ function run_silo_test { cd ../../.. AVG_ASSERT=$(echo "${COUNT_ASSERT} * 100 / ${TOTAL_RUN}" | bc -l | xargs printf "%.1f") - echo "Runs: ${TOTAL_RUN} | Assertion rate: ${AVG_ASSERT}%" + echo "Runs: ${TOTAL_RUN} | Assertions: ${COUNT_ASSERT} | Assertion rate: ${AVG_ASSERT}%" } function run_mabain_test { - export LD_LIBRARY_PATH="$${MABAINLIB}" + export LD_LIBRARY_PATH="${MABAINLIB}" - echo "Mabain assertion test" COUNT_ASSERT=0 EXE='./mb_multi_thread_insert_test_assert' @@ -54,17 +52,12 @@ function run_mabain_test { cd ../.. AVG_ASSERT=$(echo "${COUNT_ASSERT} * 100 / ${TOTAL_RUN}" | bc -l | xargs printf "%.1f") - echo "Runs: ${TOTAL_RUN} | Assertion rate: ${AVG_ASSERT}%" + echo "Runs: ${TOTAL_RUN} | Assertions: ${COUNT_ASSERT} | Assertion rate: ${AVG_ASSERT}%" } -#function run_all_tests { -# for t in ${TESTS} -# do -# echo "running ${t}" -# (run_${t}_test 2>&1) > "${t}.log" -# run_${t}_test &> "${t}.log" -# done -#} - -run_silo_test -run_mabain_test +echo "** Assertion test for some application benchmarks: ${TESTS} **" +for t in ${TESTS} +do + echo -n "${t} " + run_${t}_test +done diff --git a/scripts/test_all.sh b/scripts/app_test_all.sh similarity index 62% rename from scripts/test_all.sh rename to scripts/app_test_all.sh index 4ac1597..b21c36c 100755 --- a/scripts/test_all.sh +++ b/scripts/app_test_all.sh @@ -9,17 +9,17 @@ fi # Clear data rm *.log 2> /dev/null -echo "test application benchmarks" +echo "** Performance test for application benchmarks **" # Run in all-core configuration if [ ! -d "all-core" ]; then mkdir all-core fi -echo "running each benchmark with multiple cores for ${TOTAL_RUN} times" -rm all-core/*.log 2> /dev/null +echo "Running each benchmark with multiple cores for ${TOTAL_RUN} times" +#rm all-core/*.log 2> /dev/null ./run.sh $TOTAL_RUN mv *.log all-core -echo "done" +echo "Done" python calculator.py all-core # Run in single-core configuration @@ -27,10 +27,10 @@ if [ ! -d "single-core" ]; then mkdir single-core fi -echo "running each benchmark with a single core for ${TOTAL_RUN} times" -rm single-core/*.log 2> /dev/null +echo "Running each benchmark with a single core for ${TOTAL_RUN} times" +#rm single-core/*.log 2> /dev/null taskset -c 0 ./run.sh $TOTAL_RUN mv *.log single-core -echo "done" +echo "Done" python calculator.py single-core diff --git a/scripts/calculator.py b/scripts/calculator.py index 8989701..e2a85b8 100644 --- a/scripts/calculator.py +++ b/scripts/calculator.py @@ -3,10 +3,19 @@ import re import statistics import sys +def checkRace(content): + has_race = re.search(r'race', content) + if has_race: + print 'data race detected' + else: + print 'data race not detected' + def GdaxStatistics(filename): gdax_data = [] with open(filename, 'r') as f: content = f.read() + checkRace(content) + allruns = re.findall(r'(0-0\.49.*?19.99 s: \d+)', content, flags=re.DOTALL) for run in allruns: iterations = 0 @@ -23,6 +32,7 @@ def SiloStatistics(filename): data = [] with open(filename, 'r') as f: content = f.read() + checkRace(content) allruns = re.findall(r'agg_throughput: (\d+\.?\d*) ops', content) data = [float(x) for x in allruns] @@ -32,6 +42,7 @@ def TimeStatistics(filename): data = [] with open(filename, 'r') as f: content = f.read() + checkRace(content) allruns = re.findall(r'real.*?(\d+)m(\d+\.\d+)s', content) for run in allruns: (minute,second) = run @@ -43,6 +54,7 @@ def TimeStatistics(filename): def JsbenchStatistics(filename): with open(filename, 'r') as f: content = f.read() + checkRace(content) result = re.search(r'(Final results.*?runs)', content, flags=re.DOTALL) print(result.group(0)) diff --git a/scripts/do_test_all.sh b/scripts/do_test_all.sh new file mode 100755 index 0000000..7c806e7 --- /dev/null +++ b/scripts/do_test_all.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# Test application benchmarks +./app_test_all.sh +echo "" + +# Test CDSChecker data structure benchmarks +cd cdschecker_modified_benchmarks +./test_all.sh +cd .. +echo "" + +# Test data structures with bugs that tsan11/tsan11rec cannot detect +cd tsan11-missingbug +./test_all.sh +cd .. +echo "" + +# Test assertion failures in Silo and Mabain +./app_assertion_test.sh diff --git a/scripts/run.sh b/scripts/run.sh index 865c44a..b6be652 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -62,17 +62,12 @@ function run_jsbench_test { cd .. } -function run_all_tests { - for t in ${TESTS} - do - echo "running ${t}" - (run_${t}_test 2>&1) > "${t}.log" -# run_${t}_test &> "${t}.log" - done -} - -# Remove previous output files -rm *.log 2> /dev/null rm $REDUNDANT 2> /dev/null -run_all_tests +echo "Benchmarks: ${TESTS}" +for t in ${TESTS} +do + rm "${t}.log" 2> /dev/null + echo "Running ${t}" + (run_${t}_test 2>&1) > "${t}.log" +done diff --git a/scripts/setup.sh b/scripts/setup.sh index 9cfd1fa..ab007ea 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -7,8 +7,10 @@ git clone git://plrg.eecs.uci.edu/c11concurrency-benchmarks.git tsan11-benchmark cd tsan11-benchmarks git checkout tsan11-docker cp /data/scripts/build.sh . +cp /data/scripts/do_test_all.sh . +cp /data/scripts/app_assertion_test.sh . +cp /data/scripts/app_test_all.sh . cp /data/scripts/run.sh . -cp /data/scripts/test_all.sh . cp /data/scripts/calculator.py . ./build.sh cd .. @@ -19,8 +21,10 @@ git clone git://plrg.eecs.uci.edu/c11concurrency-benchmarks.git tsan11rec-benchm cd tsan11rec-benchmarks git checkout tsan11-docker cp /data/scripts/build.sh . +cp /data/scripts/do_test_all.sh . +cp /data/scripts/app_assertion_test.sh . +cp /data/scripts/app_test_all.sh . cp /data/scripts/run.sh . -cp /data/scripts/test_all.sh . cp /data/scripts/calculator.py . sed -i "s/tsan11/tsan11rec/g" clang clang++ gcc g++ build.sh run.sh ./build.sh -- 2.34.1