| 148 |
lars |
1 |
<?php declare(strict_types=1);
|
|
|
2 |
/*
|
|
|
3 |
* This file is part of PHPUnit.
|
|
|
4 |
*
|
|
|
5 |
* (c) Sebastian Bergmann <sebastian@phpunit.de>
|
|
|
6 |
*
|
|
|
7 |
* For the full copyright and license information, please view the LICENSE
|
|
|
8 |
* file that was distributed with this source code.
|
|
|
9 |
*/
|
|
|
10 |
namespace PHPUnit\Runner;
|
|
|
11 |
|
|
|
12 |
use function array_diff;
|
|
|
13 |
use function array_merge;
|
|
|
14 |
use function array_reverse;
|
|
|
15 |
use function array_splice;
|
|
|
16 |
use function count;
|
|
|
17 |
use function in_array;
|
|
|
18 |
use function max;
|
|
|
19 |
use function shuffle;
|
|
|
20 |
use function usort;
|
|
|
21 |
use PHPUnit\Framework\DataProviderTestSuite;
|
|
|
22 |
use PHPUnit\Framework\Reorderable;
|
|
|
23 |
use PHPUnit\Framework\Test;
|
|
|
24 |
use PHPUnit\Framework\TestCase;
|
|
|
25 |
use PHPUnit\Framework\TestSuite;
|
|
|
26 |
use PHPUnit\Util\Test as TestUtil;
|
|
|
27 |
|
|
|
28 |
/**
|
|
|
29 |
* @internal This class is not covered by the backward compatibility promise for PHPUnit
|
|
|
30 |
*/
|
|
|
31 |
final class TestSuiteSorter
|
|
|
32 |
{
|
|
|
33 |
/**
|
|
|
34 |
* @var int
|
|
|
35 |
*/
|
|
|
36 |
public const ORDER_DEFAULT = 0;
|
|
|
37 |
|
|
|
38 |
/**
|
|
|
39 |
* @var int
|
|
|
40 |
*/
|
|
|
41 |
public const ORDER_RANDOMIZED = 1;
|
|
|
42 |
|
|
|
43 |
/**
|
|
|
44 |
* @var int
|
|
|
45 |
*/
|
|
|
46 |
public const ORDER_REVERSED = 2;
|
|
|
47 |
|
|
|
48 |
/**
|
|
|
49 |
* @var int
|
|
|
50 |
*/
|
|
|
51 |
public const ORDER_DEFECTS_FIRST = 3;
|
|
|
52 |
|
|
|
53 |
/**
|
|
|
54 |
* @var int
|
|
|
55 |
*/
|
|
|
56 |
public const ORDER_DURATION = 4;
|
|
|
57 |
|
|
|
58 |
/**
|
|
|
59 |
* Order tests by @size annotation 'small', 'medium', 'large'.
|
|
|
60 |
*
|
|
|
61 |
* @var int
|
|
|
62 |
*/
|
|
|
63 |
public const ORDER_SIZE = 5;
|
|
|
64 |
|
|
|
65 |
/**
|
|
|
66 |
* List of sorting weights for all test result codes. A higher number gives higher priority.
|
|
|
67 |
*/
|
|
|
68 |
private const DEFECT_SORT_WEIGHT = [
|
|
|
69 |
BaseTestRunner::STATUS_ERROR => 6,
|
|
|
70 |
BaseTestRunner::STATUS_FAILURE => 5,
|
|
|
71 |
BaseTestRunner::STATUS_WARNING => 4,
|
|
|
72 |
BaseTestRunner::STATUS_INCOMPLETE => 3,
|
|
|
73 |
BaseTestRunner::STATUS_RISKY => 2,
|
|
|
74 |
BaseTestRunner::STATUS_SKIPPED => 1,
|
|
|
75 |
BaseTestRunner::STATUS_UNKNOWN => 0,
|
|
|
76 |
];
|
|
|
77 |
|
|
|
78 |
private const SIZE_SORT_WEIGHT = [
|
|
|
79 |
TestUtil::SMALL => 1,
|
|
|
80 |
TestUtil::MEDIUM => 2,
|
|
|
81 |
TestUtil::LARGE => 3,
|
|
|
82 |
TestUtil::UNKNOWN => 4,
|
|
|
83 |
];
|
|
|
84 |
|
|
|
85 |
/**
|
|
|
86 |
* @var array<string, int> Associative array of (string => DEFECT_SORT_WEIGHT) elements
|
|
|
87 |
*/
|
|
|
88 |
private $defectSortOrder = [];
|
|
|
89 |
|
|
|
90 |
/**
|
|
|
91 |
* @var TestResultCache
|
|
|
92 |
*/
|
|
|
93 |
private $cache;
|
|
|
94 |
|
|
|
95 |
/**
|
|
|
96 |
* @var array<string> A list of normalized names of tests before reordering
|
|
|
97 |
*/
|
|
|
98 |
private $originalExecutionOrder = [];
|
|
|
99 |
|
|
|
100 |
/**
|
|
|
101 |
* @var array<string> A list of normalized names of tests affected by reordering
|
|
|
102 |
*/
|
|
|
103 |
private $executionOrder = [];
|
|
|
104 |
|
|
|
105 |
public function __construct(?TestResultCache $cache = null)
|
|
|
106 |
{
|
|
|
107 |
$this->cache = $cache ?? new NullTestResultCache;
|
|
|
108 |
}
|
|
|
109 |
|
|
|
110 |
/**
|
|
|
111 |
* @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
|
|
|
112 |
* @throws Exception
|
|
|
113 |
*/
|
|
|
114 |
public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects, bool $isRootTestSuite = true): void
|
|
|
115 |
{
|
|
|
116 |
$allowedOrders = [
|
|
|
117 |
self::ORDER_DEFAULT,
|
|
|
118 |
self::ORDER_REVERSED,
|
|
|
119 |
self::ORDER_RANDOMIZED,
|
|
|
120 |
self::ORDER_DURATION,
|
|
|
121 |
self::ORDER_SIZE,
|
|
|
122 |
];
|
|
|
123 |
|
|
|
124 |
if (!in_array($order, $allowedOrders, true)) {
|
|
|
125 |
throw new Exception(
|
|
|
126 |
'$order must be one of TestSuiteSorter::ORDER_[DEFAULT|REVERSED|RANDOMIZED|DURATION|SIZE]'
|
|
|
127 |
);
|
|
|
128 |
}
|
|
|
129 |
|
|
|
130 |
$allowedOrderDefects = [
|
|
|
131 |
self::ORDER_DEFAULT,
|
|
|
132 |
self::ORDER_DEFECTS_FIRST,
|
|
|
133 |
];
|
|
|
134 |
|
|
|
135 |
if (!in_array($orderDefects, $allowedOrderDefects, true)) {
|
|
|
136 |
throw new Exception(
|
|
|
137 |
'$orderDefects must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_DEFECTS_FIRST'
|
|
|
138 |
);
|
|
|
139 |
}
|
|
|
140 |
|
|
|
141 |
if ($isRootTestSuite) {
|
|
|
142 |
$this->originalExecutionOrder = $this->calculateTestExecutionOrder($suite);
|
|
|
143 |
}
|
|
|
144 |
|
|
|
145 |
if ($suite instanceof TestSuite) {
|
|
|
146 |
foreach ($suite as $_suite) {
|
|
|
147 |
$this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects, false);
|
|
|
148 |
}
|
|
|
149 |
|
|
|
150 |
if ($orderDefects === self::ORDER_DEFECTS_FIRST) {
|
|
|
151 |
$this->addSuiteToDefectSortOrder($suite);
|
|
|
152 |
}
|
|
|
153 |
|
|
|
154 |
$this->sort($suite, $order, $resolveDependencies, $orderDefects);
|
|
|
155 |
}
|
|
|
156 |
|
|
|
157 |
if ($isRootTestSuite) {
|
|
|
158 |
$this->executionOrder = $this->calculateTestExecutionOrder($suite);
|
|
|
159 |
}
|
|
|
160 |
}
|
|
|
161 |
|
|
|
162 |
public function getOriginalExecutionOrder(): array
|
|
|
163 |
{
|
|
|
164 |
return $this->originalExecutionOrder;
|
|
|
165 |
}
|
|
|
166 |
|
|
|
167 |
public function getExecutionOrder(): array
|
|
|
168 |
{
|
|
|
169 |
return $this->executionOrder;
|
|
|
170 |
}
|
|
|
171 |
|
|
|
172 |
private function sort(TestSuite $suite, int $order, bool $resolveDependencies, int $orderDefects): void
|
|
|
173 |
{
|
|
|
174 |
if (empty($suite->tests())) {
|
|
|
175 |
return;
|
|
|
176 |
}
|
|
|
177 |
|
|
|
178 |
if ($order === self::ORDER_REVERSED) {
|
|
|
179 |
$suite->setTests($this->reverse($suite->tests()));
|
|
|
180 |
} elseif ($order === self::ORDER_RANDOMIZED) {
|
|
|
181 |
$suite->setTests($this->randomize($suite->tests()));
|
|
|
182 |
} elseif ($order === self::ORDER_DURATION && $this->cache !== null) {
|
|
|
183 |
$suite->setTests($this->sortByDuration($suite->tests()));
|
|
|
184 |
} elseif ($order === self::ORDER_SIZE) {
|
|
|
185 |
$suite->setTests($this->sortBySize($suite->tests()));
|
|
|
186 |
}
|
|
|
187 |
|
|
|
188 |
if ($orderDefects === self::ORDER_DEFECTS_FIRST && $this->cache !== null) {
|
|
|
189 |
$suite->setTests($this->sortDefectsFirst($suite->tests()));
|
|
|
190 |
}
|
|
|
191 |
|
|
|
192 |
if ($resolveDependencies && !($suite instanceof DataProviderTestSuite)) {
|
|
|
193 |
/** @var TestCase[] $tests */
|
|
|
194 |
$tests = $suite->tests();
|
|
|
195 |
|
|
|
196 |
$suite->setTests($this->resolveDependencies($tests));
|
|
|
197 |
}
|
|
|
198 |
}
|
|
|
199 |
|
|
|
200 |
/**
|
|
|
201 |
* @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
|
|
|
202 |
*/
|
|
|
203 |
private function addSuiteToDefectSortOrder(TestSuite $suite): void
|
|
|
204 |
{
|
|
|
205 |
$max = 0;
|
|
|
206 |
|
|
|
207 |
foreach ($suite->tests() as $test) {
|
|
|
208 |
if (!$test instanceof Reorderable) {
|
|
|
209 |
continue;
|
|
|
210 |
}
|
|
|
211 |
|
|
|
212 |
if (!isset($this->defectSortOrder[$test->sortId()])) {
|
|
|
213 |
$this->defectSortOrder[$test->sortId()] = self::DEFECT_SORT_WEIGHT[$this->cache->getState($test->sortId())];
|
|
|
214 |
$max = max($max, $this->defectSortOrder[$test->sortId()]);
|
|
|
215 |
}
|
|
|
216 |
}
|
|
|
217 |
|
|
|
218 |
$this->defectSortOrder[$suite->sortId()] = $max;
|
|
|
219 |
}
|
|
|
220 |
|
|
|
221 |
private function reverse(array $tests): array
|
|
|
222 |
{
|
|
|
223 |
return array_reverse($tests);
|
|
|
224 |
}
|
|
|
225 |
|
|
|
226 |
private function randomize(array $tests): array
|
|
|
227 |
{
|
|
|
228 |
shuffle($tests);
|
|
|
229 |
|
|
|
230 |
return $tests;
|
|
|
231 |
}
|
|
|
232 |
|
|
|
233 |
private function sortDefectsFirst(array $tests): array
|
|
|
234 |
{
|
|
|
235 |
usort(
|
|
|
236 |
$tests,
|
|
|
237 |
/**
|
|
|
238 |
* @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
|
|
|
239 |
*/
|
|
|
240 |
function ($left, $right)
|
|
|
241 |
{
|
|
|
242 |
return $this->cmpDefectPriorityAndTime($left, $right);
|
|
|
243 |
}
|
|
|
244 |
);
|
|
|
245 |
|
|
|
246 |
return $tests;
|
|
|
247 |
}
|
|
|
248 |
|
|
|
249 |
private function sortByDuration(array $tests): array
|
|
|
250 |
{
|
|
|
251 |
usort(
|
|
|
252 |
$tests,
|
|
|
253 |
/**
|
|
|
254 |
* @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
|
|
|
255 |
*/
|
|
|
256 |
function ($left, $right)
|
|
|
257 |
{
|
|
|
258 |
return $this->cmpDuration($left, $right);
|
|
|
259 |
}
|
|
|
260 |
);
|
|
|
261 |
|
|
|
262 |
return $tests;
|
|
|
263 |
}
|
|
|
264 |
|
|
|
265 |
private function sortBySize(array $tests): array
|
|
|
266 |
{
|
|
|
267 |
usort(
|
|
|
268 |
$tests,
|
|
|
269 |
/**
|
|
|
270 |
* @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
|
|
|
271 |
*/
|
|
|
272 |
function ($left, $right)
|
|
|
273 |
{
|
|
|
274 |
return $this->cmpSize($left, $right);
|
|
|
275 |
}
|
|
|
276 |
);
|
|
|
277 |
|
|
|
278 |
return $tests;
|
|
|
279 |
}
|
|
|
280 |
|
|
|
281 |
/**
|
|
|
282 |
* Comparator callback function to sort tests for "reach failure as fast as possible".
|
|
|
283 |
*
|
|
|
284 |
* 1. sort tests by defect weight defined in self::DEFECT_SORT_WEIGHT
|
|
|
285 |
* 2. when tests are equally defective, sort the fastest to the front
|
|
|
286 |
* 3. do not reorder successful tests
|
|
|
287 |
*
|
|
|
288 |
* @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
|
|
|
289 |
*/
|
|
|
290 |
private function cmpDefectPriorityAndTime(Test $a, Test $b): int
|
|
|
291 |
{
|
|
|
292 |
if (!($a instanceof Reorderable && $b instanceof Reorderable)) {
|
|
|
293 |
return 0;
|
|
|
294 |
}
|
|
|
295 |
|
|
|
296 |
$priorityA = $this->defectSortOrder[$a->sortId()] ?? 0;
|
|
|
297 |
$priorityB = $this->defectSortOrder[$b->sortId()] ?? 0;
|
|
|
298 |
|
|
|
299 |
if ($priorityB <=> $priorityA) {
|
|
|
300 |
// Sort defect weight descending
|
|
|
301 |
return $priorityB <=> $priorityA;
|
|
|
302 |
}
|
|
|
303 |
|
|
|
304 |
if ($priorityA || $priorityB) {
|
|
|
305 |
return $this->cmpDuration($a, $b);
|
|
|
306 |
}
|
|
|
307 |
|
|
|
308 |
// do not change execution order
|
|
|
309 |
return 0;
|
|
|
310 |
}
|
|
|
311 |
|
|
|
312 |
/**
|
|
|
313 |
* Compares test duration for sorting tests by duration ascending.
|
|
|
314 |
*
|
|
|
315 |
* @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
|
|
|
316 |
*/
|
|
|
317 |
private function cmpDuration(Test $a, Test $b): int
|
|
|
318 |
{
|
|
|
319 |
if (!($a instanceof Reorderable && $b instanceof Reorderable)) {
|
|
|
320 |
return 0;
|
|
|
321 |
}
|
|
|
322 |
|
|
|
323 |
return $this->cache->getTime($a->sortId()) <=> $this->cache->getTime($b->sortId());
|
|
|
324 |
}
|
|
|
325 |
|
|
|
326 |
/**
|
|
|
327 |
* Compares test size for sorting tests small->medium->large->unknown.
|
|
|
328 |
*/
|
|
|
329 |
private function cmpSize(Test $a, Test $b): int
|
|
|
330 |
{
|
|
|
331 |
$sizeA = ($a instanceof TestCase || $a instanceof DataProviderTestSuite)
|
|
|
332 |
? $a->getSize()
|
|
|
333 |
: TestUtil::UNKNOWN;
|
|
|
334 |
$sizeB = ($b instanceof TestCase || $b instanceof DataProviderTestSuite)
|
|
|
335 |
? $b->getSize()
|
|
|
336 |
: TestUtil::UNKNOWN;
|
|
|
337 |
|
|
|
338 |
return self::SIZE_SORT_WEIGHT[$sizeA] <=> self::SIZE_SORT_WEIGHT[$sizeB];
|
|
|
339 |
}
|
|
|
340 |
|
|
|
341 |
/**
|
|
|
342 |
* Reorder Tests within a TestCase in such a way as to resolve as many dependencies as possible.
|
|
|
343 |
* The algorithm will leave the tests in original running order when it can.
|
|
|
344 |
* For more details see the documentation for test dependencies.
|
|
|
345 |
*
|
|
|
346 |
* Short description of algorithm:
|
|
|
347 |
* 1. Pick the next Test from remaining tests to be checked for dependencies.
|
|
|
348 |
* 2. If the test has no dependencies: mark done, start again from the top
|
|
|
349 |
* 3. If the test has dependencies but none left to do: mark done, start again from the top
|
|
|
350 |
* 4. When we reach the end add any leftover tests to the end. These will be marked 'skipped' during execution.
|
|
|
351 |
*
|
|
|
352 |
* @param array<DataProviderTestSuite|TestCase> $tests
|
|
|
353 |
*
|
|
|
354 |
* @return array<DataProviderTestSuite|TestCase>
|
|
|
355 |
*/
|
|
|
356 |
private function resolveDependencies(array $tests): array
|
|
|
357 |
{
|
|
|
358 |
$newTestOrder = [];
|
|
|
359 |
$i = 0;
|
|
|
360 |
$provided = [];
|
|
|
361 |
|
|
|
362 |
do {
|
|
|
363 |
if ([] === array_diff($tests[$i]->requires(), $provided)) {
|
|
|
364 |
$provided = array_merge($provided, $tests[$i]->provides());
|
|
|
365 |
$newTestOrder = array_merge($newTestOrder, array_splice($tests, $i, 1));
|
|
|
366 |
$i = 0;
|
|
|
367 |
} else {
|
|
|
368 |
$i++;
|
|
|
369 |
}
|
|
|
370 |
} while (!empty($tests) && ($i < count($tests)));
|
|
|
371 |
|
|
|
372 |
return array_merge($newTestOrder, $tests);
|
|
|
373 |
}
|
|
|
374 |
|
|
|
375 |
/**
|
|
|
376 |
* @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
|
|
|
377 |
*/
|
|
|
378 |
private function calculateTestExecutionOrder(Test $suite): array
|
|
|
379 |
{
|
|
|
380 |
$tests = [];
|
|
|
381 |
|
|
|
382 |
if ($suite instanceof TestSuite) {
|
|
|
383 |
foreach ($suite->tests() as $test) {
|
|
|
384 |
if (!$test instanceof TestSuite && $test instanceof Reorderable) {
|
|
|
385 |
$tests[] = $test->sortId();
|
|
|
386 |
} else {
|
|
|
387 |
$tests = array_merge($tests, $this->calculateTestExecutionOrder($test));
|
|
|
388 |
}
|
|
|
389 |
}
|
|
|
390 |
}
|
|
|
391 |
|
|
|
392 |
return $tests;
|
|
|
393 |
}
|
|
|
394 |
}
|