View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.plugin.surefire.report;
20  
21  import java.io.File;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.TreeMap;
28  import java.util.concurrent.ConcurrentLinkedQueue;
29  
30  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
31  import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
32  import org.apache.maven.plugin.surefire.log.api.Level;
33  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
34  import org.apache.maven.surefire.api.report.ReporterFactory;
35  import org.apache.maven.surefire.api.report.StackTraceWriter;
36  import org.apache.maven.surefire.api.report.TestOutputReportEntry;
37  import org.apache.maven.surefire.api.report.TestReportListener;
38  import org.apache.maven.surefire.api.suite.RunResult;
39  import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener;
40  import org.apache.maven.surefire.extensions.StatelessReportEventListener;
41  import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
42  import org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener;
43  import org.apache.maven.surefire.report.RunStatistics;
44  import org.apache.maven.surefire.shared.utils.StringUtils;
45  import org.apache.maven.surefire.shared.utils.logging.MessageBuilder;
46  
47  import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel;
48  import static org.apache.maven.plugin.surefire.report.ConsoleReporter.PLAIN;
49  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.ERROR;
50  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FAILURE;
51  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FLAKE;
52  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SKIPPED;
53  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SUCCESS;
54  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.UNKNOWN;
55  import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
56  import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
57  import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
58  import static org.apache.maven.surefire.api.util.internal.ObjectUtils.useNonNull;
59  import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer;
60  
61  /**
62   * Provides reporting modules on the plugin side.
63   * <br>
64   * Keeps a centralized count of test run results.
65   *
66   * @author Kristian Rosenvold
67   */
68  public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
69      private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<>();
70      private final StartupReportConfiguration reportConfiguration;
71      private final ConsoleLogger consoleLogger;
72      private final Integer forkNumber;
73  
74      private RunStatistics globalStats = new RunStatistics();
75  
76      // from "<testclass>.<testmethod>" -> statistics about all the runs for success tests
77      private Map<String, List<TestMethodStats>> successTests;
78  
79      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
80      private Map<String, List<TestMethodStats>> flakyTests;
81  
82      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
83      private Map<String, List<TestMethodStats>> failedTests;
84  
85      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
86      private Map<String, List<TestMethodStats>> errorTests;
87  
88      public DefaultReporterFactory(StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger) {
89          this(reportConfiguration, consoleLogger, null);
90      }
91  
92      public DefaultReporterFactory(
93              StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger, Integer forkNumber) {
94          this.reportConfiguration = reportConfiguration;
95          this.consoleLogger = consoleLogger;
96          this.forkNumber = forkNumber;
97      }
98  
99      @Override
100     public TestReportListener<TestOutputReportEntry> createTestReportListener() {
101         TestSetRunListener testSetRunListener = new TestSetRunListener(
102                 createConsoleReporter(),
103                 createFileReporter(),
104                 createSimpleXMLReporter(),
105                 createConsoleOutputReceiver(),
106                 createStatisticsReporter(),
107                 reportConfiguration.isTrimStackTrace(),
108                 PLAIN.equals(reportConfiguration.getReportFormat()),
109                 reportConfiguration.isBriefOrPlainFormat(),
110                 consoleLogger,
111                 reportConfiguration.getReporterFactoryOptions().isStatPerSourceName());
112         addListener(testSetRunListener);
113         return testSetRunListener;
114     }
115 
116     @Override
117     public File getReportsDirectory() {
118         return reportConfiguration.getReportsDirectory();
119     }
120 
121     private StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> createConsoleReporter() {
122         StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> consoleReporter =
123                 reportConfiguration.instantiateConsoleReporter(consoleLogger);
124         return useNonNull(consoleReporter, NullConsoleReporter.INSTANCE);
125     }
126 
127     private StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> createFileReporter() {
128         StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> fileReporter =
129                 reportConfiguration.instantiateFileReporter(forkNumber);
130         return useNonNull(fileReporter, NullFileReporter.INSTANCE);
131     }
132 
133     private StatelessReportEventListener<WrappedReportEntry, TestSetStats> createSimpleXMLReporter() {
134         StatelessReportEventListener<WrappedReportEntry, TestSetStats> xmlReporter =
135                 reportConfiguration.instantiateStatelessXmlReporter(forkNumber);
136         return useNonNull(xmlReporter, NullStatelessXmlReporter.INSTANCE);
137     }
138 
139     private ConsoleOutputReportEventListener createConsoleOutputReceiver() {
140         ConsoleOutputReportEventListener outputReporter =
141                 reportConfiguration.instantiateConsoleOutputFileReporter(forkNumber);
142         return useNonNull(outputReporter, NullConsoleOutputReceiver.INSTANCE);
143     }
144 
145     private StatisticsReporter createStatisticsReporter() {
146         StatisticsReporter statisticsReporter = reportConfiguration.getStatisticsReporter();
147         return useNonNull(statisticsReporter, NullStatisticsReporter.INSTANCE);
148     }
149 
150     @Override
151     public void mergeFromOtherFactories(Collection<DefaultReporterFactory> factories) {
152         for (DefaultReporterFactory factory : factories) {
153             listeners.addAll(factory.listeners);
154         }
155     }
156 
157     final void addListener(TestSetRunListener listener) {
158         listeners.add(listener);
159     }
160 
161     @Override
162     public RunResult close() {
163         mergeTestHistoryResult();
164         runCompleted();
165         for (TestSetRunListener listener : listeners) {
166             listener.close();
167         }
168         return globalStats.getRunResult();
169     }
170 
171     @Override
172     public void runStarting() {
173         if (reportConfiguration.isPrintSummary()) {
174             log("");
175             log("-------------------------------------------------------");
176             log(" T E S T S");
177             log("-------------------------------------------------------");
178         }
179     }
180 
181     private void runCompleted() {
182         if (reportConfiguration.isPrintSummary()) {
183             log("");
184             log("Results:");
185             log("");
186         }
187         boolean printedFailures = printTestFailures(TestResultType.FAILURE);
188         boolean printedErrors = printTestFailures(TestResultType.ERROR);
189         boolean printedFlakes = printTestFailures(TestResultType.FLAKE);
190         if (reportConfiguration.isPrintSummary()) {
191             if (printedFailures | printedErrors | printedFlakes) {
192                 log("");
193             }
194             boolean hasSuccessful = globalStats.getCompletedCount() > 0;
195             boolean hasSkipped = globalStats.getSkipped() > 0;
196             log(globalStats.getSummary(), hasSuccessful, printedFailures, printedErrors, hasSkipped, printedFlakes);
197             log("");
198         }
199     }
200 
201     public RunStatistics getGlobalRunStatistics() {
202         mergeTestHistoryResult();
203         return globalStats;
204     }
205 
206     /**
207      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
208      * if it only has errors or failures, then count its result based on its first run.
209      *
210      * @param reportEntries the list of test run report type for a given test
211      * @param rerunFailingTestsCount configured rerun count for failing tests
212      * @return the type of test result
213      */
214     // Use default visibility for testing
215     static TestResultType getTestResultType(List<ReportEntryType> reportEntries, int rerunFailingTestsCount) {
216         if (reportEntries == null || reportEntries.isEmpty()) {
217             return UNKNOWN;
218         }
219 
220         boolean seenSuccess = false, seenFailure = false, seenError = false;
221         for (ReportEntryType resultType : reportEntries) {
222             if (resultType == ReportEntryType.SUCCESS) {
223                 seenSuccess = true;
224             } else if (resultType == ReportEntryType.FAILURE) {
225                 seenFailure = true;
226             } else if (resultType == ReportEntryType.ERROR) {
227                 seenError = true;
228             }
229         }
230 
231         if (seenFailure || seenError) {
232             if (seenSuccess && rerunFailingTestsCount > 0) {
233                 return TestResultType.FLAKE;
234             } else {
235                 return seenError ? TestResultType.ERROR : TestResultType.FAILURE;
236             }
237         } else if (seenSuccess) {
238             return TestResultType.SUCCESS;
239         } else {
240             return SKIPPED;
241         }
242     }
243 
244     /**
245      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
246      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
247      */
248     private void mergeTestHistoryResult() {
249         globalStats = new RunStatistics();
250         successTests = new TreeMap<>();
251         flakyTests = new TreeMap<>();
252         failedTests = new TreeMap<>();
253         errorTests = new TreeMap<>();
254 
255         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<>();
256         // Merge all the stats for tests from listeners
257         for (TestSetRunListener listener : listeners) {
258             for (TestMethodStats methodStats : listener.getTestMethodStats()) {
259                 List<TestMethodStats> currentMethodStats =
260                         mergedTestHistoryResult.get(methodStats.getTestClassMethodName());
261                 if (currentMethodStats == null) {
262                     currentMethodStats = new ArrayList<>();
263                     currentMethodStats.add(methodStats);
264                     mergedTestHistoryResult.put(methodStats.getTestClassMethodName(), currentMethodStats);
265                 } else {
266                     currentMethodStats.add(methodStats);
267                 }
268             }
269         }
270 
271         // Update globalStatistics by iterating through mergedTestHistoryResult
272         int completedCount = 0, skipped = 0;
273         Map<String, List<TestMethodStats>> beforeAllFailures = new HashMap<>();
274 
275         for (Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet()) {
276             List<TestMethodStats> testMethodStats = entry.getValue();
277             String testClassMethodName = entry.getKey();
278 
279             // Handle @BeforeAll failures (null, empty, ends with ".null" or ".initializationError" method names)
280             // But only if they actually failed (ERROR or FAILURE), not if they were skipped
281             if ((StringUtils.isBlank(testClassMethodName)
282                             || testClassMethodName.endsWith(".null")
283                             || testClassMethodName.endsWith(".initializationError"))
284                     && (testClassMethodName == null || !testClassMethodName.contains("$"))) {
285 
286                 // Check if this is actually a failure/error (not skipped or success)
287                 boolean isActualFailure = testMethodStats.stream()
288                         .anyMatch(stat -> stat.getResultType() == ReportEntryType.ERROR
289                                 || stat.getResultType() == ReportEntryType.FAILURE);
290 
291                 if (isActualFailure) {
292                     // Extract class name from the test class method name
293                     String className = extractClassNameFromMethodName(testClassMethodName);
294                     if (className != null) {
295                         if (beforeAllFailures.containsKey(className)) {
296                             List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
297                             previousMethodStats.addAll(testMethodStats);
298                             beforeAllFailures.put(className, previousMethodStats);
299                         } else {
300                             beforeAllFailures.put(className, new ArrayList<>(testMethodStats));
301                         }
302                     }
303                     // Skip normal processing of @BeforeAll failures because it needs special care
304                     continue;
305                 }
306                 // If it's skipped or success with null method name, fall through to normal processing
307             }
308 
309             completedCount++;
310 
311             List<ReportEntryType> resultTypes = new ArrayList<>();
312             for (TestMethodStats methodStats : testMethodStats) {
313                 resultTypes.add(methodStats.getResultType());
314             }
315 
316             switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) {
317                 case SUCCESS:
318                     // If there are multiple successful runs of the same test, count all of them
319                     int successCount = 0;
320                     for (ReportEntryType type : resultTypes) {
321                         if (type == ReportEntryType.SUCCESS) {
322                             successCount++;
323                         }
324                     }
325                     completedCount += successCount - 1;
326                     successTests.put(testClassMethodName, testMethodStats);
327                     break;
328                 case SKIPPED:
329                     skipped++;
330                     break;
331                 case FLAKE:
332                     flakyTests.put(testClassMethodName, testMethodStats);
333                     break;
334                 case FAILURE:
335                     failedTests.put(testClassMethodName, testMethodStats);
336                     break;
337                 case ERROR:
338                     errorTests.put(testClassMethodName, testMethodStats);
339                     break;
340                 default:
341                     throw new IllegalStateException("Get unknown test result type");
342             }
343         }
344 
345         // Loop over all success tests and find those that are passed flakes for beforeAll failures
346         for (Map.Entry<String, List<TestMethodStats>> entry : successTests.entrySet()) {
347             List<TestMethodStats> testMethodStats = entry.getValue();
348             String testClassMethodName = entry.getKey();
349             // If current test belong to class that failed during beforeAll store that info to proper log info
350             String className = extractClassNameFromMethodName(testClassMethodName);
351             if (beforeAllFailures.containsKey(className)) {
352                 List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
353                 previousMethodStats.addAll(testMethodStats);
354                 beforeAllFailures.put(className, previousMethodStats);
355             }
356         }
357 
358         // Process @BeforeAll failures after we know which classes have successful tests
359         for (Map.Entry<String, List<TestMethodStats>> entry : beforeAllFailures.entrySet()) {
360             String className = entry.getKey();
361             List<TestMethodStats> testMethodStats = entry.getValue();
362             String classNameKey = className + ".<beforeAll>";
363 
364             if (reportConfiguration.getRerunFailingTestsCount() > 0
365                     && testMethodStats.stream()
366                             .anyMatch(methodStats -> methodStats.getTestClassMethodName() != null
367                                     && !methodStats.getTestClassMethodName().isEmpty()
368                                     && methodStats.getResultType().equals(ReportEntryType.SUCCESS))) {
369                 flakyTests.put(classNameKey, testMethodStats);
370             } else {
371                 errorTests.put(classNameKey, testMethodStats);
372                 completedCount++;
373             }
374         }
375 
376         globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size());
377     }
378 
379     /**
380      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
381      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test.
382      *
383      * @param type   the type of results to be printed, could be error, failure or flake
384      * @return {@code true} if printed some lines
385      */
386     // Use default visibility for testing
387     boolean printTestFailures(TestResultType type) {
388         final Map<String, List<TestMethodStats>> testStats;
389         final Level level;
390         switch (type) {
391             case FAILURE:
392                 testStats = failedTests;
393                 level = Level.FAILURE;
394                 break;
395             case ERROR:
396                 testStats = errorTests;
397                 level = Level.FAILURE;
398                 break;
399             case FLAKE:
400                 testStats = flakyTests;
401                 level = Level.UNSTABLE;
402                 break;
403             default:
404                 return false;
405         }
406 
407         boolean printed = false;
408         if (!testStats.isEmpty()) {
409             log(type.getLogPrefix(), level);
410             printed = true;
411         }
412 
413         for (Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet()) {
414             List<TestMethodStats> testMethodStats = entry.getValue();
415             if (testMethodStats.size() == 1) {
416                 // No rerun, follow the original output format
417                 failure("  " + testMethodStats.get(0).getStackTraceWriter().smartTrimmedStackTrace());
418             } else {
419                 log(entry.getKey(), level);
420                 for (int i = 0; i < testMethodStats.size(); i++) {
421                     StackTraceWriter failureStackTrace = testMethodStats.get(i).getStackTraceWriter();
422                     if (failureStackTrace == null) {
423                         success("  Run " + (i + 1) + ": PASS");
424                     } else {
425                         failure("  Run " + (i + 1) + ": " + failureStackTrace.smartTrimmedStackTrace());
426                     }
427                 }
428                 log("");
429             }
430         }
431         return printed;
432     }
433 
434     /**
435      * Extract class name from test class method name like "com.example.TestClass.methodName"
436      */
437     private static String extractClassNameFromMethodName(String testClassMethodName) {
438         if (StringUtils.isBlank(testClassMethodName)) {
439             return null;
440         }
441         int lastDotIndex = testClassMethodName.lastIndexOf('.');
442         if (lastDotIndex > 0) {
443             return testClassMethodName.substring(0, lastDotIndex);
444         }
445         return null;
446     }
447 
448     /**
449      * Extract class name from stack trace when method name is null (e.g., @BeforeAll failures)
450      */
451     private static String extractClassNameFromStackTrace(TestMethodStats stats) {
452         if (stats.getStackTraceWriter() == null) {
453             return null;
454         }
455         String stackTrace = stats.getStackTraceWriter().smartTrimmedStackTrace();
456         if (stackTrace == null || stackTrace.isEmpty()) {
457             return null;
458         }
459 
460         // Strip everything after the first whitespace character
461         int firstWhitespace = stackTrace.indexOf(' ');
462         if (firstWhitespace > 0) {
463             stackTrace = stackTrace.substring(0, firstWhitespace);
464         }
465 
466         return extractClassNameFromMethodName(stackTrace);
467     }
468 
469     // Describe the result of a given test
470     enum TestResultType {
471         ERROR("Errors: "),
472         FAILURE("Failures: "),
473         FLAKE("Flakes: "),
474         SUCCESS("Success: "),
475         SKIPPED("Skipped: "),
476         UNKNOWN("Unknown: ");
477 
478         private final String logPrefix;
479 
480         TestResultType(String logPrefix) {
481             this.logPrefix = logPrefix;
482         }
483 
484         public String getLogPrefix() {
485             return logPrefix;
486         }
487     }
488 
489     private void log(String s, boolean success, boolean failures, boolean errors, boolean skipped, boolean flakes) {
490         Level level = resolveLevel(success, failures, errors, skipped, flakes);
491         log(s, level);
492     }
493 
494     private void log(String s, Level level) {
495         switch (level) {
496             case FAILURE:
497                 failure(s);
498                 break;
499             case UNSTABLE:
500                 warning(s);
501                 break;
502             case SUCCESS:
503                 success(s);
504                 break;
505             default:
506                 info(s);
507         }
508     }
509 
510     private void log(String s) {
511         consoleLogger.info(s);
512     }
513 
514     private void info(String s) {
515         MessageBuilder builder = buffer();
516         consoleLogger.info(builder.a(s).toString());
517     }
518 
519     private void warning(String s) {
520         MessageBuilder builder = buffer();
521         consoleLogger.warning(builder.warning(s).toString());
522     }
523 
524     private void success(String s) {
525         MessageBuilder builder = buffer();
526         consoleLogger.info(builder.success(s).toString());
527     }
528 
529     private void failure(String s) {
530         MessageBuilder builder = buffer();
531         consoleLogger.error(builder.failure(s).toString());
532     }
533 }