[lit] Add basic flaky test retry functionality

The plan is to use this for the sanitizer test suite on Windows.  See
PR24554 for more details on why we need this.

Tested manually by injecting rand() into a sanitizer test and watching
what it does.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@246704 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/utils/lit/lit/Test.py b/utils/lit/lit/Test.py
index 38bb41b0252d59a3769a3be2097cbae659f6d109..701335541fb7c82c8e1a5db0d555e0a38e1dcc5b 100644 (file)
--- a/utils/lit/lit/Test.py
+++ b/utils/lit/lit/Test.py
@@ -27,6 +27,7 @@ class ResultCode(object):
                          (self.name, self.isFailure))
 
 PASS        = ResultCode('PASS', False)
+FLAKYPASS   = ResultCode('FLAKYPASS', False)
 XFAIL       = ResultCode('XFAIL', False)
 FAIL        = ResultCode('FAIL', True)
 XPASS       = ResultCode('XPASS', True)
@@ -253,4 +254,4 @@ class Test:
             xml += "\n\t</failure>\n</testcase>"
         else:
             xml += "/>"
-        return xml
\ No newline at end of file
+        return xml

diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py
index 055a282f3bca11902d0a1a793746b6cc3f77f465..24075ff94048e4667001db56bb1dff03ee732948 100644 (file)
--- a/utils/lit/lit/TestRunner.py
+++ b/utils/lit/lit/TestRunner.py
@@ -602,5 +602,17 @@ def executeShTest(test, litConfig, useExternalSh,
         return lit.Test.Result(Test.PASS)
 
     script, tmpBase, execdir = res
-    return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
 
+    # Re-run failed tests up to test_retry_attempts times.
+    attempts = 1
+    if hasattr(test.config, 'test_retry_attempts'):
+        attempts += test.config.test_retry_attempts
+    for i in range(attempts):
+        res = _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
+        if res.code != Test.FAIL:
+            break
+    # If we had to run the test more than once, count it as a flaky pass. These
+    # will be printed separately in the test summary.
+    if i > 0 and res.code == Test.PASS:
+        res.code = Test.FLAKYPASS
+    return res

diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py
index e3722674f63f2820e70302ddb334d0589e98ba65..630cb54f96d5f162f5187921e1ed5547f1d98d5f 100755 (executable)
--- a/utils/lit/lit/main.py
+++ b/utils/lit/lit/main.py
@@ -414,6 +414,7 @@ def main(builtinParameters = {}):
         lit.util.printHistogram(test_times, title='Tests')
 
     for name,code in (('Expected Passes    ', lit.Test.PASS),
+                      ('Passes With Retry  ', lit.Test.FLAKYPASS),
                       ('Expected Failures  ', lit.Test.XFAIL),
                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),