From: Reid Kleckner <rnk@google.com>
Date: Wed, 2 Sep 2015 20:32:41 +0000 (+0000)
Subject: [lit] Add basic flaky test retry functionality
X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=5ef0a653de6cb33afbf694ff6f402e9767ad126a;p=oota-llvm.git

[lit] Add basic flaky test retry functionality

The plan is to use this for the sanitizer test suite on Windows.  See
PR24554 for more details on why we need this.

Tested manually by injecting rand() into a sanitizer test and watching
what it does.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@246704 91177308-0d34-0410-b5e6-96231b3b80d8
---

diff --git a/utils/lit/lit/Test.py b/utils/lit/lit/Test.py
index 38bb41b0252..701335541fb 100644
--- a/utils/lit/lit/Test.py
+++ b/utils/lit/lit/Test.py
@@ -27,6 +27,7 @@ class ResultCode(object):
                          (self.name, self.isFailure))
 
 PASS        = ResultCode('PASS', False)
+FLAKYPASS   = ResultCode('FLAKYPASS', False)
 XFAIL       = ResultCode('XFAIL', False)
 FAIL        = ResultCode('FAIL', True)
 XPASS       = ResultCode('XPASS', True)
@@ -253,4 +254,4 @@ class Test:
             xml += "\n\t</failure>\n</testcase>"
         else:
             xml += "/>"
-        return xml
\ No newline at end of file
+        return xml
diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py
index 055a282f3bc..24075ff9404 100644
--- a/utils/lit/lit/TestRunner.py
+++ b/utils/lit/lit/TestRunner.py
@@ -602,5 +602,17 @@ def executeShTest(test, litConfig, useExternalSh,
         return lit.Test.Result(Test.PASS)
 
     script, tmpBase, execdir = res
-    return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
 
+    # Re-run failed tests up to test_retry_attempts times.
+    attempts = 1
+    if hasattr(test.config, 'test_retry_attempts'):
+        attempts += test.config.test_retry_attempts
+    for i in range(attempts):
+        res = _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
+        if res.code != Test.FAIL:
+            break
+    # If we had to run the test more than once, count it as a flaky pass. These
+    # will be printed separately in the test summary.
+    if i > 0 and res.code == Test.PASS:
+        res.code = Test.FLAKYPASS
+    return res
diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py
index e3722674f63..630cb54f96d 100755
--- a/utils/lit/lit/main.py
+++ b/utils/lit/lit/main.py
@@ -414,6 +414,7 @@ def main(builtinParameters = {}):
         lit.util.printHistogram(test_times, title='Tests')
 
     for name,code in (('Expected Passes    ', lit.Test.PASS),
+                      ('Passes With Retry  ', lit.Test.FLAKYPASS),
                       ('Expected Failures  ', lit.Test.XFAIL),
                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),