1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.plugin.surefire.report;
20
21 import java.io.File;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.TreeMap;
28 import java.util.concurrent.ConcurrentLinkedQueue;
29
30 import org.apache.maven.plugin.surefire.StartupReportConfiguration;
31 import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
32 import org.apache.maven.plugin.surefire.log.api.Level;
33 import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
34 import org.apache.maven.surefire.api.report.ReporterFactory;
35 import org.apache.maven.surefire.api.report.StackTraceWriter;
36 import org.apache.maven.surefire.api.report.TestOutputReportEntry;
37 import org.apache.maven.surefire.api.report.TestReportListener;
38 import org.apache.maven.surefire.api.suite.RunResult;
39 import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener;
40 import org.apache.maven.surefire.extensions.StatelessReportEventListener;
41 import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
42 import org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener;
43 import org.apache.maven.surefire.report.RunStatistics;
44 import org.apache.maven.surefire.shared.utils.StringUtils;
45 import org.apache.maven.surefire.shared.utils.logging.MessageBuilder;
46
47 import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel;
48 import static org.apache.maven.plugin.surefire.report.ConsoleReporter.PLAIN;
49 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.ERROR;
50 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FAILURE;
51 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FLAKE;
52 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SKIPPED;
53 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SUCCESS;
54 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.UNKNOWN;
55 import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
56 import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
57 import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
58 import static org.apache.maven.surefire.api.util.internal.ObjectUtils.useNonNull;
59 import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer;
60
61
62
63
64
65
66
67
68 public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
69 private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<>();
70 private final StartupReportConfiguration reportConfiguration;
71 private final ConsoleLogger consoleLogger;
72 private final Integer forkNumber;
73
74 private RunStatistics globalStats = new RunStatistics();
75
76
77 private Map<String, List<TestMethodStats>> successTests;
78
79
80 private Map<String, List<TestMethodStats>> flakyTests;
81
82
83 private Map<String, List<TestMethodStats>> failedTests;
84
85
86 private Map<String, List<TestMethodStats>> errorTests;
87
88 public DefaultReporterFactory(StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger) {
89 this(reportConfiguration, consoleLogger, null);
90 }
91
92 public DefaultReporterFactory(
93 StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger, Integer forkNumber) {
94 this.reportConfiguration = reportConfiguration;
95 this.consoleLogger = consoleLogger;
96 this.forkNumber = forkNumber;
97 }
98
99 @Override
100 public TestReportListener<TestOutputReportEntry> createTestReportListener() {
101 TestSetRunListener testSetRunListener = new TestSetRunListener(
102 createConsoleReporter(),
103 createFileReporter(),
104 createSimpleXMLReporter(),
105 createConsoleOutputReceiver(),
106 createStatisticsReporter(),
107 reportConfiguration.isTrimStackTrace(),
108 PLAIN.equals(reportConfiguration.getReportFormat()),
109 reportConfiguration.isBriefOrPlainFormat(),
110 consoleLogger,
111 reportConfiguration.getReporterFactoryOptions().isStatPerSourceName());
112 addListener(testSetRunListener);
113 return testSetRunListener;
114 }
115
116 @Override
117 public File getReportsDirectory() {
118 return reportConfiguration.getReportsDirectory();
119 }
120
121 private StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> createConsoleReporter() {
122 StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> consoleReporter =
123 reportConfiguration.instantiateConsoleReporter(consoleLogger);
124 return useNonNull(consoleReporter, NullConsoleReporter.INSTANCE);
125 }
126
127 private StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> createFileReporter() {
128 StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> fileReporter =
129 reportConfiguration.instantiateFileReporter(forkNumber);
130 return useNonNull(fileReporter, NullFileReporter.INSTANCE);
131 }
132
133 private StatelessReportEventListener<WrappedReportEntry, TestSetStats> createSimpleXMLReporter() {
134 StatelessReportEventListener<WrappedReportEntry, TestSetStats> xmlReporter =
135 reportConfiguration.instantiateStatelessXmlReporter(forkNumber);
136 return useNonNull(xmlReporter, NullStatelessXmlReporter.INSTANCE);
137 }
138
139 private ConsoleOutputReportEventListener createConsoleOutputReceiver() {
140 ConsoleOutputReportEventListener outputReporter =
141 reportConfiguration.instantiateConsoleOutputFileReporter(forkNumber);
142 return useNonNull(outputReporter, NullConsoleOutputReceiver.INSTANCE);
143 }
144
145 private StatisticsReporter createStatisticsReporter() {
146 StatisticsReporter statisticsReporter = reportConfiguration.getStatisticsReporter();
147 return useNonNull(statisticsReporter, NullStatisticsReporter.INSTANCE);
148 }
149
150 @Override
151 public void mergeFromOtherFactories(Collection<DefaultReporterFactory> factories) {
152 for (DefaultReporterFactory factory : factories) {
153 listeners.addAll(factory.listeners);
154 }
155 }
156
157 final void addListener(TestSetRunListener listener) {
158 listeners.add(listener);
159 }
160
161 @Override
162 public RunResult close() {
163 mergeTestHistoryResult();
164 runCompleted();
165 for (TestSetRunListener listener : listeners) {
166 listener.close();
167 }
168 return globalStats.getRunResult();
169 }
170
171 @Override
172 public void runStarting() {
173 if (reportConfiguration.isPrintSummary()) {
174 log("");
175 log("-------------------------------------------------------");
176 log(" T E S T S");
177 log("-------------------------------------------------------");
178 }
179 }
180
181 private void runCompleted() {
182 if (reportConfiguration.isPrintSummary()) {
183 log("");
184 log("Results:");
185 log("");
186 }
187 boolean printedFailures = printTestFailures(TestResultType.FAILURE);
188 boolean printedErrors = printTestFailures(TestResultType.ERROR);
189 boolean printedFlakes = printTestFailures(TestResultType.FLAKE);
190 if (reportConfiguration.isPrintSummary()) {
191 if (printedFailures | printedErrors | printedFlakes) {
192 log("");
193 }
194 boolean hasSuccessful = globalStats.getCompletedCount() > 0;
195 boolean hasSkipped = globalStats.getSkipped() > 0;
196 log(globalStats.getSummary(), hasSuccessful, printedFailures, printedErrors, hasSkipped, printedFlakes);
197 log("");
198 }
199 }
200
201 public RunStatistics getGlobalRunStatistics() {
202 mergeTestHistoryResult();
203 return globalStats;
204 }
205
206
207
208
209
210
211
212
213
214
215 static TestResultType getTestResultType(List<ReportEntryType> reportEntries, int rerunFailingTestsCount) {
216 if (reportEntries == null || reportEntries.isEmpty()) {
217 return UNKNOWN;
218 }
219
220 boolean seenSuccess = false, seenFailure = false, seenError = false;
221 for (ReportEntryType resultType : reportEntries) {
222 if (resultType == ReportEntryType.SUCCESS) {
223 seenSuccess = true;
224 } else if (resultType == ReportEntryType.FAILURE) {
225 seenFailure = true;
226 } else if (resultType == ReportEntryType.ERROR) {
227 seenError = true;
228 }
229 }
230
231 if (seenFailure || seenError) {
232 if (seenSuccess && rerunFailingTestsCount > 0) {
233 return TestResultType.FLAKE;
234 } else {
235 return seenError ? TestResultType.ERROR : TestResultType.FAILURE;
236 }
237 } else if (seenSuccess) {
238 return TestResultType.SUCCESS;
239 } else {
240 return SKIPPED;
241 }
242 }
243
244
245
246
247
248 private void mergeTestHistoryResult() {
249 globalStats = new RunStatistics();
250 successTests = new TreeMap<>();
251 flakyTests = new TreeMap<>();
252 failedTests = new TreeMap<>();
253 errorTests = new TreeMap<>();
254
255 Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<>();
256
257 for (TestSetRunListener listener : listeners) {
258 for (TestMethodStats methodStats : listener.getTestMethodStats()) {
259 List<TestMethodStats> currentMethodStats =
260 mergedTestHistoryResult.get(methodStats.getTestClassMethodName());
261 if (currentMethodStats == null) {
262 currentMethodStats = new ArrayList<>();
263 currentMethodStats.add(methodStats);
264 mergedTestHistoryResult.put(methodStats.getTestClassMethodName(), currentMethodStats);
265 } else {
266 currentMethodStats.add(methodStats);
267 }
268 }
269 }
270
271
272 int completedCount = 0, skipped = 0;
273 Map<String, List<TestMethodStats>> beforeAllFailures = new HashMap<>();
274
275 for (Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet()) {
276 List<TestMethodStats> testMethodStats = entry.getValue();
277 String testClassMethodName = entry.getKey();
278
279
280
281 if ((StringUtils.isBlank(testClassMethodName) || testClassMethodName.endsWith(".null"))
282 && (testClassMethodName == null || !testClassMethodName.contains("$"))) {
283
284
285 boolean isActualFailure = testMethodStats.stream()
286 .anyMatch(stat -> stat.getResultType() == ReportEntryType.ERROR
287 || stat.getResultType() == ReportEntryType.FAILURE);
288
289 if (isActualFailure) {
290
291 String className = extractClassNameFromMethodName(testClassMethodName);
292 if (className != null) {
293 if (beforeAllFailures.containsKey(className)) {
294 List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
295 previousMethodStats.addAll(testMethodStats);
296 beforeAllFailures.put(className, previousMethodStats);
297 } else {
298 beforeAllFailures.put(className, new ArrayList<>(testMethodStats));
299 }
300 }
301
302 continue;
303 }
304
305 }
306
307 completedCount++;
308
309 List<ReportEntryType> resultTypes = new ArrayList<>();
310 for (TestMethodStats methodStats : testMethodStats) {
311 resultTypes.add(methodStats.getResultType());
312 }
313
314 switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) {
315 case SUCCESS:
316
317 int successCount = 0;
318 for (ReportEntryType type : resultTypes) {
319 if (type == ReportEntryType.SUCCESS) {
320 successCount++;
321 }
322 }
323 completedCount += successCount - 1;
324 successTests.put(testClassMethodName, testMethodStats);
325 break;
326 case SKIPPED:
327 skipped++;
328 break;
329 case FLAKE:
330 flakyTests.put(testClassMethodName, testMethodStats);
331 break;
332 case FAILURE:
333 failedTests.put(testClassMethodName, testMethodStats);
334 break;
335 case ERROR:
336 errorTests.put(testClassMethodName, testMethodStats);
337 break;
338 default:
339 throw new IllegalStateException("Get unknown test result type");
340 }
341 }
342
343
344 for (Map.Entry<String, List<TestMethodStats>> entry : successTests.entrySet()) {
345 List<TestMethodStats> testMethodStats = entry.getValue();
346 String testClassMethodName = entry.getKey();
347
348 String className = extractClassNameFromMethodName(testClassMethodName);
349 if (beforeAllFailures.containsKey(className)) {
350 List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
351 previousMethodStats.addAll(testMethodStats);
352 beforeAllFailures.put(className, previousMethodStats);
353 }
354 }
355
356
357 for (Map.Entry<String, List<TestMethodStats>> entry : beforeAllFailures.entrySet()) {
358 String className = entry.getKey();
359 List<TestMethodStats> testMethodStats = entry.getValue();
360 String classNameKey = className + ".<beforeAll>";
361
362 if (reportConfiguration.getRerunFailingTestsCount() > 0
363 && testMethodStats.stream()
364 .anyMatch(methodStats -> methodStats.getTestClassMethodName() != null
365 && !methodStats.getTestClassMethodName().isEmpty()
366 && methodStats.getResultType().equals(ReportEntryType.SUCCESS))) {
367 flakyTests.put(classNameKey, testMethodStats);
368 } else {
369 errorTests.put(classNameKey, testMethodStats);
370 completedCount++;
371 }
372 }
373
374 globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size());
375 }
376
377
378
379
380
381
382
383
384
385 boolean printTestFailures(TestResultType type) {
386 final Map<String, List<TestMethodStats>> testStats;
387 final Level level;
388 switch (type) {
389 case FAILURE:
390 testStats = failedTests;
391 level = Level.FAILURE;
392 break;
393 case ERROR:
394 testStats = errorTests;
395 level = Level.FAILURE;
396 break;
397 case FLAKE:
398 testStats = flakyTests;
399 level = Level.UNSTABLE;
400 break;
401 default:
402 return false;
403 }
404
405 boolean printed = false;
406 if (!testStats.isEmpty()) {
407 log(type.getLogPrefix(), level);
408 printed = true;
409 }
410
411 for (Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet()) {
412 List<TestMethodStats> testMethodStats = entry.getValue();
413 if (testMethodStats.size() == 1) {
414
415 failure(" " + testMethodStats.get(0).getStackTraceWriter().smartTrimmedStackTrace());
416 } else {
417 log(entry.getKey(), level);
418 for (int i = 0; i < testMethodStats.size(); i++) {
419 StackTraceWriter failureStackTrace = testMethodStats.get(i).getStackTraceWriter();
420 if (failureStackTrace == null) {
421 success(" Run " + (i + 1) + ": PASS");
422 } else {
423 failure(" Run " + (i + 1) + ": " + failureStackTrace.smartTrimmedStackTrace());
424 }
425 }
426 log("");
427 }
428 }
429 return printed;
430 }
431
432
433
434
435 private static String extractClassNameFromMethodName(String testClassMethodName) {
436 if (StringUtils.isBlank(testClassMethodName)) {
437 return null;
438 }
439 int lastDotIndex = testClassMethodName.lastIndexOf('.');
440 if (lastDotIndex > 0) {
441 return testClassMethodName.substring(0, lastDotIndex);
442 }
443 return null;
444 }
445
446
447
448
449 private static String extractClassNameFromStackTrace(TestMethodStats stats) {
450 if (stats.getStackTraceWriter() == null) {
451 return null;
452 }
453 String stackTrace = stats.getStackTraceWriter().smartTrimmedStackTrace();
454 if (stackTrace == null || stackTrace.isEmpty()) {
455 return null;
456 }
457
458
459 int firstWhitespace = stackTrace.indexOf(' ');
460 if (firstWhitespace > 0) {
461 stackTrace = stackTrace.substring(0, firstWhitespace);
462 }
463
464 return extractClassNameFromMethodName(stackTrace);
465 }
466
467
468 enum TestResultType {
469 ERROR("Errors: "),
470 FAILURE("Failures: "),
471 FLAKE("Flakes: "),
472 SUCCESS("Success: "),
473 SKIPPED("Skipped: "),
474 UNKNOWN("Unknown: ");
475
476 private final String logPrefix;
477
478 TestResultType(String logPrefix) {
479 this.logPrefix = logPrefix;
480 }
481
482 public String getLogPrefix() {
483 return logPrefix;
484 }
485 }
486
487 private void log(String s, boolean success, boolean failures, boolean errors, boolean skipped, boolean flakes) {
488 Level level = resolveLevel(success, failures, errors, skipped, flakes);
489 log(s, level);
490 }
491
492 private void log(String s, Level level) {
493 switch (level) {
494 case FAILURE:
495 failure(s);
496 break;
497 case UNSTABLE:
498 warning(s);
499 break;
500 case SUCCESS:
501 success(s);
502 break;
503 default:
504 info(s);
505 }
506 }
507
508 private void log(String s) {
509 consoleLogger.info(s);
510 }
511
512 private void info(String s) {
513 MessageBuilder builder = buffer();
514 consoleLogger.info(builder.a(s).toString());
515 }
516
517 private void warning(String s) {
518 MessageBuilder builder = buffer();
519 consoleLogger.warning(builder.warning(s).toString());
520 }
521
522 private void success(String s) {
523 MessageBuilder builder = buffer();
524 consoleLogger.info(builder.success(s).toString());
525 }
526
527 private void failure(String s) {
528 MessageBuilder builder = buffer();
529 consoleLogger.error(builder.failure(s).toString());
530 }
531 }