2 from xml.sax.saxutils import escape
3 from json import JSONEncoder
7 class ResultCode(object):
8 """Test result codes."""
10 # We override __new__ and __getnewargs__ to ensure that pickling still
11 # provides unique ResultCode objects in any particular instance.
13 def __new__(cls, name, isFailure):
14 res = cls._instances.get(name)
16 cls._instances[name] = res = super(ResultCode, cls).__new__(cls)
18 def __getnewargs__(self):
19 return (self.name, self.isFailure)
21 def __init__(self, name, isFailure):
23 self.isFailure = isFailure
26 return '%s%r' % (self.__class__.__name__,
27 (self.name, self.isFailure))
29 PASS = ResultCode('PASS', False)
30 XFAIL = ResultCode('XFAIL', False)
31 FAIL = ResultCode('FAIL', True)
32 XPASS = ResultCode('XPASS', True)
33 UNRESOLVED = ResultCode('UNRESOLVED', True)
34 UNSUPPORTED = ResultCode('UNSUPPORTED', False)
38 class MetricValue(object):
43 Convert this metric to a string suitable for displaying as part of the
46 raise RuntimeError("abstract method")
50 todata() -> json-serializable data
52 Convert this metric to content suitable for serializing in the JSON test
55 raise RuntimeError("abstract method")
57 class IntMetricValue(MetricValue):
58 def __init__(self, value):
62 return str(self.value)
67 class RealMetricValue(MetricValue):
68 def __init__(self, value):
72 return '%.4f' % self.value
77 class JSONMetricValue(MetricValue):
79 JSONMetricValue is used for types that are representable in the output
80 but that are otherwise uninterpreted.
82 def __init__(self, value):
83 # Ensure the value is a serializable by trying to encode it.
84 # WARNING: The value may change before it is encoded again, and may
85 # not be encodable after the change.
94 e = JSONEncoder(indent=2, sort_keys=True)
95 return e.encode(self.value)
100 def toMetricValue(value):
101 if isinstance(value, MetricValue):
103 elif isinstance(value, int) or isinstance(value, long):
104 return IntMetricValue(value)
105 elif isinstance(value, float):
106 return RealMetricValue(value)
108 # Try to create a JSONMetricValue and let the constructor throw
109 # if value is not a valid type.
110 return JSONMetricValue(value)
115 class Result(object):
116 """Wrapper for the results of executing an individual test."""
118 def __init__(self, code, output='', elapsed=None):
123 # The wall timing to execute the test, if timing.
124 self.elapsed = elapsed
125 # The metrics reported by this test.
128 def addMetric(self, name, value):
130 addMetric(name, value)
132 Attach a test metric to the test result, with the given name and list of
133 values. It is an error to attempt to attach the metrics with the same
136 Each value must be an instance of a MetricValue subclass.
138 if name in self.metrics:
139 raise ValueError("result already includes metrics for %r" % (
141 if not isinstance(value, MetricValue):
142 raise TypeError("unexpected metric value: %r" % (value,))
143 self.metrics[name] = value
148 """TestSuite - Information on a group of tests.
150 A test suite groups together a set of logically related tests.
153 def __init__(self, name, source_root, exec_root, config):
155 self.source_root = source_root
156 self.exec_root = exec_root
157 # The test suite configuration.
160 def getSourcePath(self, components):
161 return os.path.join(self.source_root, *components)
163 def getExecPath(self, components):
164 return os.path.join(self.exec_root, *components)
167 """Test - Information on a single test instance."""
169 def __init__(self, suite, path_in_suite, config, file_path = None):
171 self.path_in_suite = path_in_suite
173 self.file_path = file_path
174 # A list of conditions under which this test is expected to fail. These
175 # can optionally be provided by test format handlers, and will be
176 # honored when the test result is supplied.
178 # The test result, once complete.
181 def setResult(self, result):
182 if self.result is not None:
183 raise ArgumentError("test result already set")
184 if not isinstance(result, Result):
185 raise ArgumentError("unexpected result type")
189 # Apply the XFAIL handling to resolve the result exit code.
190 if self.isExpectedToFail():
191 if self.result.code == PASS:
192 self.result.code = XPASS
193 elif self.result.code == FAIL:
194 self.result.code = XFAIL
196 def getFullName(self):
197 return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
199 def getFilePath(self):
201 return self.file_path
202 return self.getSourcePath()
204 def getSourcePath(self):
205 return self.suite.getSourcePath(self.path_in_suite)
207 def getExecPath(self):
208 return self.suite.getExecPath(self.path_in_suite)
210 def isExpectedToFail(self):
212 isExpectedToFail() -> bool
214 Check whether this test is expected to fail in the current
215 configuration. This check relies on the test xfails property which by
216 some test formats may not be computed until the test has first been
220 # Check if any of the xfails match an available feature or the target.
221 for item in self.xfails:
222 # If this is the wildcard, it always fails.
226 # If this is an exact match for one of the features, it fails.
227 if item in self.config.available_features:
230 # If this is a part of the target triple, it fails.
231 if item in self.suite.config.target_triple:
237 def getJUnitXML(self):
238 test_name = self.path_in_suite[-1]
239 test_path = self.path_in_suite[:-1]
240 safe_test_path = [x.replace(".","_") for x in test_path]
241 safe_name = self.suite.name.replace(".","-")
244 class_name = safe_name + "." + "/".join(safe_test_path)
246 class_name = safe_name + "." + safe_name
248 xml = "<testcase classname='" + class_name + "' name='" + \
250 xml += " time='%.2f'" % (self.result.elapsed,)
251 if self.result.code.isFailure:
252 xml += ">\n\t<failure >\n" + escape(self.result.output)
253 xml += "\n\t</failure>\n</testcase>"