1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.plugin.surefire.report;
20
21 import java.io.File;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.TreeMap;
28 import java.util.concurrent.ConcurrentLinkedQueue;
29
30 import org.apache.maven.plugin.surefire.StartupReportConfiguration;
31 import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
32 import org.apache.maven.plugin.surefire.log.api.Level;
33 import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
34 import org.apache.maven.surefire.api.report.ReporterFactory;
35 import org.apache.maven.surefire.api.report.StackTraceWriter;
36 import org.apache.maven.surefire.api.report.TestOutputReportEntry;
37 import org.apache.maven.surefire.api.report.TestReportListener;
38 import org.apache.maven.surefire.api.suite.RunResult;
39 import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener;
40 import org.apache.maven.surefire.extensions.StatelessReportEventListener;
41 import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
42 import org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener;
43 import org.apache.maven.surefire.report.RunStatistics;
44 import org.apache.maven.surefire.shared.utils.StringUtils;
45 import org.apache.maven.surefire.shared.utils.logging.MessageBuilder;
46
47 import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel;
48 import static org.apache.maven.plugin.surefire.report.ConsoleReporter.PLAIN;
49 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.ERROR;
50 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FAILURE;
51 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FLAKE;
52 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SKIPPED;
53 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SUCCESS;
54 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.UNKNOWN;
55 import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
56 import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
57 import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
58 import static org.apache.maven.surefire.api.util.internal.ObjectUtils.useNonNull;
59 import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer;
60
61
62
63
64
65
66
67
68 public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
69 private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<>();
70 private final StartupReportConfiguration reportConfiguration;
71 private final ConsoleLogger consoleLogger;
72 private final Integer forkNumber;
73
74 private RunStatistics globalStats = new RunStatistics();
75
76
77 private Map<String, List<TestMethodStats>> successTests;
78
79
80 private Map<String, List<TestMethodStats>> flakyTests;
81
82
83 private Map<String, List<TestMethodStats>> failedTests;
84
85
86 private Map<String, List<TestMethodStats>> errorTests;
87
88 public DefaultReporterFactory(StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger) {
89 this(reportConfiguration, consoleLogger, null);
90 }
91
92 public DefaultReporterFactory(
93 StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger, Integer forkNumber) {
94 this.reportConfiguration = reportConfiguration;
95 this.consoleLogger = consoleLogger;
96 this.forkNumber = forkNumber;
97 }
98
99 @Override
100 public TestReportListener<TestOutputReportEntry> createTestReportListener() {
101 TestSetRunListener testSetRunListener = new TestSetRunListener(
102 createConsoleReporter(),
103 createFileReporter(),
104 createSimpleXMLReporter(),
105 createConsoleOutputReceiver(),
106 createStatisticsReporter(),
107 reportConfiguration.isTrimStackTrace(),
108 PLAIN.equals(reportConfiguration.getReportFormat()),
109 reportConfiguration.isBriefOrPlainFormat(),
110 consoleLogger,
111 reportConfiguration.getReporterFactoryOptions().isStatPerSourceName());
112 addListener(testSetRunListener);
113 return testSetRunListener;
114 }
115
116 @Override
117 public File getReportsDirectory() {
118 return reportConfiguration.getReportsDirectory();
119 }
120
121 private StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> createConsoleReporter() {
122 StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> consoleReporter =
123 reportConfiguration.instantiateConsoleReporter(consoleLogger);
124 return useNonNull(consoleReporter, NullConsoleReporter.INSTANCE);
125 }
126
127 private StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> createFileReporter() {
128 StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> fileReporter =
129 reportConfiguration.instantiateFileReporter(forkNumber);
130 return useNonNull(fileReporter, NullFileReporter.INSTANCE);
131 }
132
133 private StatelessReportEventListener<WrappedReportEntry, TestSetStats> createSimpleXMLReporter() {
134 StatelessReportEventListener<WrappedReportEntry, TestSetStats> xmlReporter =
135 reportConfiguration.instantiateStatelessXmlReporter(forkNumber);
136 return useNonNull(xmlReporter, NullStatelessXmlReporter.INSTANCE);
137 }
138
139 private ConsoleOutputReportEventListener createConsoleOutputReceiver() {
140 ConsoleOutputReportEventListener outputReporter =
141 reportConfiguration.instantiateConsoleOutputFileReporter(forkNumber);
142 return useNonNull(outputReporter, NullConsoleOutputReceiver.INSTANCE);
143 }
144
145 private StatisticsReporter createStatisticsReporter() {
146 StatisticsReporter statisticsReporter = reportConfiguration.getStatisticsReporter();
147 return useNonNull(statisticsReporter, NullStatisticsReporter.INSTANCE);
148 }
149
150 @Override
151 public void mergeFromOtherFactories(Collection<DefaultReporterFactory> factories) {
152 for (DefaultReporterFactory factory : factories) {
153 listeners.addAll(factory.listeners);
154 }
155 }
156
157 final void addListener(TestSetRunListener listener) {
158 listeners.add(listener);
159 }
160
161 @Override
162 public RunResult close() {
163 mergeTestHistoryResult();
164 runCompleted();
165 for (TestSetRunListener listener : listeners) {
166 listener.close();
167 }
168 return globalStats.getRunResult();
169 }
170
171 @Override
172 public void runStarting() {
173 if (reportConfiguration.isPrintSummary()) {
174 log("");
175 log("-------------------------------------------------------");
176 log(" T E S T S");
177 log("-------------------------------------------------------");
178 }
179 }
180
181 private void runCompleted() {
182 if (reportConfiguration.isPrintSummary()) {
183 log("");
184 log("Results:");
185 log("");
186 }
187 boolean printedFailures = printTestFailures(TestResultType.FAILURE);
188 boolean printedErrors = printTestFailures(TestResultType.ERROR);
189 boolean printedFlakes = printTestFailures(TestResultType.FLAKE);
190 if (reportConfiguration.isPrintSummary()) {
191 if (printedFailures | printedErrors | printedFlakes) {
192 log("");
193 }
194 boolean hasSuccessful = globalStats.getCompletedCount() > 0;
195 boolean hasSkipped = globalStats.getSkipped() > 0;
196 log(globalStats.getSummary(), hasSuccessful, printedFailures, printedErrors, hasSkipped, printedFlakes);
197 log("");
198 }
199 }
200
201 public RunStatistics getGlobalRunStatistics() {
202 mergeTestHistoryResult();
203 return globalStats;
204 }
205
206
207
208
209
210
211
212
213
214
215 static TestResultType getTestResultType(List<ReportEntryType> reportEntries, int rerunFailingTestsCount) {
216 if (reportEntries == null || reportEntries.isEmpty()) {
217 return UNKNOWN;
218 }
219
220 boolean seenSuccess = false, seenFailure = false, seenError = false;
221 for (ReportEntryType resultType : reportEntries) {
222 if (resultType == ReportEntryType.SUCCESS) {
223 seenSuccess = true;
224 } else if (resultType == ReportEntryType.FAILURE) {
225 seenFailure = true;
226 } else if (resultType == ReportEntryType.ERROR) {
227 seenError = true;
228 }
229 }
230
231 if (seenFailure || seenError) {
232 if (seenSuccess && rerunFailingTestsCount > 0) {
233 return TestResultType.FLAKE;
234 } else {
235 return seenError ? TestResultType.ERROR : TestResultType.FAILURE;
236 }
237 } else if (seenSuccess) {
238 return TestResultType.SUCCESS;
239 } else {
240 return SKIPPED;
241 }
242 }
243
244
245
246
247
248 private void mergeTestHistoryResult() {
249 globalStats = new RunStatistics();
250 successTests = new TreeMap<>();
251 flakyTests = new TreeMap<>();
252 failedTests = new TreeMap<>();
253 errorTests = new TreeMap<>();
254
255 Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<>();
256
257 for (TestSetRunListener listener : listeners) {
258 for (TestMethodStats methodStats : listener.getTestMethodStats()) {
259 List<TestMethodStats> currentMethodStats =
260 mergedTestHistoryResult.get(methodStats.getTestClassMethodName());
261 if (currentMethodStats == null) {
262 currentMethodStats = new ArrayList<>();
263 currentMethodStats.add(methodStats);
264 mergedTestHistoryResult.put(methodStats.getTestClassMethodName(), currentMethodStats);
265 } else {
266 currentMethodStats.add(methodStats);
267 }
268 }
269 }
270
271
272 int completedCount = 0, skipped = 0;
273 Map<String, List<TestMethodStats>> beforeAllFailures = new HashMap<>();
274
275 for (Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet()) {
276 List<TestMethodStats> testMethodStats = entry.getValue();
277 String testClassMethodName = entry.getKey();
278
279
280
281 if ((StringUtils.isBlank(testClassMethodName)
282 || testClassMethodName.endsWith(".null")
283 || testClassMethodName.endsWith(".initializationError"))
284 && (testClassMethodName == null || !testClassMethodName.contains("$"))) {
285
286
287 boolean isActualFailure = testMethodStats.stream()
288 .anyMatch(stat -> stat.getResultType() == ReportEntryType.ERROR
289 || stat.getResultType() == ReportEntryType.FAILURE);
290
291 if (isActualFailure) {
292
293 String className = extractClassNameFromMethodName(testClassMethodName);
294 if (className != null) {
295 if (beforeAllFailures.containsKey(className)) {
296 List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
297 previousMethodStats.addAll(testMethodStats);
298 beforeAllFailures.put(className, previousMethodStats);
299 } else {
300 beforeAllFailures.put(className, new ArrayList<>(testMethodStats));
301 }
302 }
303
304 continue;
305 }
306
307 }
308
309 completedCount++;
310
311 List<ReportEntryType> resultTypes = new ArrayList<>();
312 for (TestMethodStats methodStats : testMethodStats) {
313 resultTypes.add(methodStats.getResultType());
314 }
315
316 switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) {
317 case SUCCESS:
318
319 int successCount = 0;
320 for (ReportEntryType type : resultTypes) {
321 if (type == ReportEntryType.SUCCESS) {
322 successCount++;
323 }
324 }
325 completedCount += successCount - 1;
326 successTests.put(testClassMethodName, testMethodStats);
327 break;
328 case SKIPPED:
329 skipped++;
330 break;
331 case FLAKE:
332 flakyTests.put(testClassMethodName, testMethodStats);
333 break;
334 case FAILURE:
335 failedTests.put(testClassMethodName, testMethodStats);
336 break;
337 case ERROR:
338 errorTests.put(testClassMethodName, testMethodStats);
339 break;
340 default:
341 throw new IllegalStateException("Get unknown test result type");
342 }
343 }
344
345
346 for (Map.Entry<String, List<TestMethodStats>> entry : successTests.entrySet()) {
347 List<TestMethodStats> testMethodStats = entry.getValue();
348 String testClassMethodName = entry.getKey();
349
350 String className = extractClassNameFromMethodName(testClassMethodName);
351 if (beforeAllFailures.containsKey(className)) {
352 List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
353 previousMethodStats.addAll(testMethodStats);
354 beforeAllFailures.put(className, previousMethodStats);
355 }
356 }
357
358
359 for (Map.Entry<String, List<TestMethodStats>> entry : beforeAllFailures.entrySet()) {
360 String className = entry.getKey();
361 List<TestMethodStats> testMethodStats = entry.getValue();
362 String classNameKey = className + ".<beforeAll>";
363
364 if (reportConfiguration.getRerunFailingTestsCount() > 0
365 && testMethodStats.stream()
366 .anyMatch(methodStats -> methodStats.getTestClassMethodName() != null
367 && !methodStats.getTestClassMethodName().isEmpty()
368 && methodStats.getResultType().equals(ReportEntryType.SUCCESS))) {
369 flakyTests.put(classNameKey, testMethodStats);
370 } else {
371 errorTests.put(classNameKey, testMethodStats);
372 completedCount++;
373 }
374 }
375
376 globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size());
377 }
378
379
380
381
382
383
384
385
386
387 boolean printTestFailures(TestResultType type) {
388 final Map<String, List<TestMethodStats>> testStats;
389 final Level level;
390 switch (type) {
391 case FAILURE:
392 testStats = failedTests;
393 level = Level.FAILURE;
394 break;
395 case ERROR:
396 testStats = errorTests;
397 level = Level.FAILURE;
398 break;
399 case FLAKE:
400 testStats = flakyTests;
401 level = Level.UNSTABLE;
402 break;
403 default:
404 return false;
405 }
406
407 boolean printed = false;
408 if (!testStats.isEmpty()) {
409 log(type.getLogPrefix(), level);
410 printed = true;
411 }
412
413 for (Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet()) {
414 List<TestMethodStats> testMethodStats = entry.getValue();
415 if (testMethodStats.size() == 1) {
416
417 failure(" " + testMethodStats.get(0).getStackTraceWriter().smartTrimmedStackTrace());
418 } else {
419 log(entry.getKey(), level);
420 for (int i = 0; i < testMethodStats.size(); i++) {
421 StackTraceWriter failureStackTrace = testMethodStats.get(i).getStackTraceWriter();
422 if (failureStackTrace == null) {
423 success(" Run " + (i + 1) + ": PASS");
424 } else {
425 failure(" Run " + (i + 1) + ": " + failureStackTrace.smartTrimmedStackTrace());
426 }
427 }
428 log("");
429 }
430 }
431 return printed;
432 }
433
434
435
436
437 private static String extractClassNameFromMethodName(String testClassMethodName) {
438 if (StringUtils.isBlank(testClassMethodName)) {
439 return null;
440 }
441 int lastDotIndex = testClassMethodName.lastIndexOf('.');
442 if (lastDotIndex > 0) {
443 return testClassMethodName.substring(0, lastDotIndex);
444 }
445 return null;
446 }
447
448
449
450
451 private static String extractClassNameFromStackTrace(TestMethodStats stats) {
452 if (stats.getStackTraceWriter() == null) {
453 return null;
454 }
455 String stackTrace = stats.getStackTraceWriter().smartTrimmedStackTrace();
456 if (stackTrace == null || stackTrace.isEmpty()) {
457 return null;
458 }
459
460
461 int firstWhitespace = stackTrace.indexOf(' ');
462 if (firstWhitespace > 0) {
463 stackTrace = stackTrace.substring(0, firstWhitespace);
464 }
465
466 return extractClassNameFromMethodName(stackTrace);
467 }
468
469
470 enum TestResultType {
471 ERROR("Errors: "),
472 FAILURE("Failures: "),
473 FLAKE("Flakes: "),
474 SUCCESS("Success: "),
475 SKIPPED("Skipped: "),
476 UNKNOWN("Unknown: ");
477
478 private final String logPrefix;
479
480 TestResultType(String logPrefix) {
481 this.logPrefix = logPrefix;
482 }
483
484 public String getLogPrefix() {
485 return logPrefix;
486 }
487 }
488
489 private void log(String s, boolean success, boolean failures, boolean errors, boolean skipped, boolean flakes) {
490 Level level = resolveLevel(success, failures, errors, skipped, flakes);
491 log(s, level);
492 }
493
494 private void log(String s, Level level) {
495 switch (level) {
496 case FAILURE:
497 failure(s);
498 break;
499 case UNSTABLE:
500 warning(s);
501 break;
502 case SUCCESS:
503 success(s);
504 break;
505 default:
506 info(s);
507 }
508 }
509
510 private void log(String s) {
511 consoleLogger.info(s);
512 }
513
514 private void info(String s) {
515 MessageBuilder builder = buffer();
516 consoleLogger.info(builder.a(s).toString());
517 }
518
519 private void warning(String s) {
520 MessageBuilder builder = buffer();
521 consoleLogger.warning(builder.warning(s).toString());
522 }
523
524 private void success(String s) {
525 MessageBuilder builder = buffer();
526 consoleLogger.info(builder.success(s).toString());
527 }
528
529 private void failure(String s) {
530 MessageBuilder builder = buffer();
531 consoleLogger.error(builder.failure(s).toString());
532 }
533 }