View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.plugin.surefire.report;
20  
21  import java.io.File;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.TreeMap;
28  import java.util.concurrent.ConcurrentLinkedQueue;
29  
30  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
31  import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
32  import org.apache.maven.plugin.surefire.log.api.Level;
33  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
34  import org.apache.maven.surefire.api.report.ReporterFactory;
35  import org.apache.maven.surefire.api.report.StackTraceWriter;
36  import org.apache.maven.surefire.api.report.TestOutputReportEntry;
37  import org.apache.maven.surefire.api.report.TestReportListener;
38  import org.apache.maven.surefire.api.suite.RunResult;
39  import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener;
40  import org.apache.maven.surefire.extensions.StatelessReportEventListener;
41  import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
42  import org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener;
43  import org.apache.maven.surefire.report.RunStatistics;
44  import org.apache.maven.surefire.shared.utils.StringUtils;
45  import org.apache.maven.surefire.shared.utils.logging.MessageBuilder;
46  
47  import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel;
48  import static org.apache.maven.plugin.surefire.report.ConsoleReporter.PLAIN;
49  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.ERROR;
50  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FAILURE;
51  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FLAKE;
52  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SKIPPED;
53  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SUCCESS;
54  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.UNKNOWN;
55  import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
56  import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
57  import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
58  import static org.apache.maven.surefire.api.util.internal.ObjectUtils.useNonNull;
59  import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer;
60  
61  /**
62   * Provides reporting modules on the plugin side.
63   * <br>
64   * Keeps a centralized count of test run results.
65   *
66   * @author Kristian Rosenvold
67   */
68  public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
69      private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<>();
70      private final StartupReportConfiguration reportConfiguration;
71      private final ConsoleLogger consoleLogger;
72      private final Integer forkNumber;
73  
74      private RunStatistics globalStats = new RunStatistics();
75  
76      // from "<testclass>.<testmethod>" -> statistics about all the runs for success tests
77      private Map<String, List<TestMethodStats>> successTests;
78  
79      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
80      private Map<String, List<TestMethodStats>> flakyTests;
81  
82      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
83      private Map<String, List<TestMethodStats>> failedTests;
84  
85      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
86      private Map<String, List<TestMethodStats>> errorTests;
87  
88      public DefaultReporterFactory(StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger) {
89          this(reportConfiguration, consoleLogger, null);
90      }
91  
92      public DefaultReporterFactory(
93              StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger, Integer forkNumber) {
94          this.reportConfiguration = reportConfiguration;
95          this.consoleLogger = consoleLogger;
96          this.forkNumber = forkNumber;
97      }
98  
99      @Override
100     public TestReportListener<TestOutputReportEntry> createTestReportListener() {
101         TestSetRunListener testSetRunListener = new TestSetRunListener(
102                 createConsoleReporter(),
103                 createFileReporter(),
104                 createSimpleXMLReporter(),
105                 createConsoleOutputReceiver(),
106                 createStatisticsReporter(),
107                 reportConfiguration.isTrimStackTrace(),
108                 PLAIN.equals(reportConfiguration.getReportFormat()),
109                 reportConfiguration.isBriefOrPlainFormat(),
110                 consoleLogger,
111                 reportConfiguration.getReporterFactoryOptions().isStatPerSourceName());
112         addListener(testSetRunListener);
113         return testSetRunListener;
114     }
115 
116     @Override
117     public File getReportsDirectory() {
118         return reportConfiguration.getReportsDirectory();
119     }
120 
121     private StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> createConsoleReporter() {
122         StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> consoleReporter =
123                 reportConfiguration.instantiateConsoleReporter(consoleLogger);
124         return useNonNull(consoleReporter, NullConsoleReporter.INSTANCE);
125     }
126 
127     private StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> createFileReporter() {
128         StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> fileReporter =
129                 reportConfiguration.instantiateFileReporter(forkNumber);
130         return useNonNull(fileReporter, NullFileReporter.INSTANCE);
131     }
132 
133     private StatelessReportEventListener<WrappedReportEntry, TestSetStats> createSimpleXMLReporter() {
134         StatelessReportEventListener<WrappedReportEntry, TestSetStats> xmlReporter =
135                 reportConfiguration.instantiateStatelessXmlReporter(forkNumber);
136         return useNonNull(xmlReporter, NullStatelessXmlReporter.INSTANCE);
137     }
138 
139     private ConsoleOutputReportEventListener createConsoleOutputReceiver() {
140         ConsoleOutputReportEventListener outputReporter =
141                 reportConfiguration.instantiateConsoleOutputFileReporter(forkNumber);
142         return useNonNull(outputReporter, NullConsoleOutputReceiver.INSTANCE);
143     }
144 
145     private StatisticsReporter createStatisticsReporter() {
146         StatisticsReporter statisticsReporter = reportConfiguration.getStatisticsReporter();
147         return useNonNull(statisticsReporter, NullStatisticsReporter.INSTANCE);
148     }
149 
150     @Override
151     public void mergeFromOtherFactories(Collection<DefaultReporterFactory> factories) {
152         for (DefaultReporterFactory factory : factories) {
153             listeners.addAll(factory.listeners);
154         }
155     }
156 
157     final void addListener(TestSetRunListener listener) {
158         listeners.add(listener);
159     }
160 
161     @Override
162     public RunResult close() {
163         mergeTestHistoryResult();
164         runCompleted();
165         for (TestSetRunListener listener : listeners) {
166             listener.close();
167         }
168         return globalStats.getRunResult();
169     }
170 
171     @Override
172     public void runStarting() {
173         if (reportConfiguration.isPrintSummary()) {
174             log("");
175             log("-------------------------------------------------------");
176             log(" T E S T S");
177             log("-------------------------------------------------------");
178         }
179     }
180 
181     private void runCompleted() {
182         if (reportConfiguration.isPrintSummary()) {
183             log("");
184             log("Results:");
185             log("");
186         }
187         boolean printedFailures = printTestFailures(TestResultType.FAILURE);
188         boolean printedErrors = printTestFailures(TestResultType.ERROR);
189         boolean printedFlakes = printTestFailures(TestResultType.FLAKE);
190         if (reportConfiguration.isPrintSummary()) {
191             if (printedFailures | printedErrors | printedFlakes) {
192                 log("");
193             }
194             boolean hasSuccessful = globalStats.getCompletedCount() > 0;
195             boolean hasSkipped = globalStats.getSkipped() > 0;
196             log(globalStats.getSummary(), hasSuccessful, printedFailures, printedErrors, hasSkipped, printedFlakes);
197             log("");
198         }
199     }
200 
201     public RunStatistics getGlobalRunStatistics() {
202         mergeTestHistoryResult();
203         return globalStats;
204     }
205 
206     /**
207      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
208      * if it only has errors or failures, then count its result based on its first run.
209      *
210      * @param reportEntries the list of test run report type for a given test
211      * @param rerunFailingTestsCount configured rerun count for failing tests
212      * @return the type of test result
213      */
214     // Use default visibility for testing
215     static TestResultType getTestResultType(List<ReportEntryType> reportEntries, int rerunFailingTestsCount) {
216         if (reportEntries == null || reportEntries.isEmpty()) {
217             return UNKNOWN;
218         }
219 
220         boolean seenSuccess = false, seenFailure = false, seenError = false;
221         for (ReportEntryType resultType : reportEntries) {
222             if (resultType == ReportEntryType.SUCCESS) {
223                 seenSuccess = true;
224             } else if (resultType == ReportEntryType.FAILURE) {
225                 seenFailure = true;
226             } else if (resultType == ReportEntryType.ERROR) {
227                 seenError = true;
228             }
229         }
230 
231         if (seenFailure || seenError) {
232             if (seenSuccess && rerunFailingTestsCount > 0) {
233                 return TestResultType.FLAKE;
234             } else {
235                 return seenError ? TestResultType.ERROR : TestResultType.FAILURE;
236             }
237         } else if (seenSuccess) {
238             return TestResultType.SUCCESS;
239         } else {
240             return SKIPPED;
241         }
242     }
243 
244     /**
245      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
246      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
247      */
248     private void mergeTestHistoryResult() {
249         globalStats = new RunStatistics();
250         successTests = new TreeMap<>();
251         flakyTests = new TreeMap<>();
252         failedTests = new TreeMap<>();
253         errorTests = new TreeMap<>();
254 
255         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<>();
256         // Merge all the stats for tests from listeners
257         for (TestSetRunListener listener : listeners) {
258             for (TestMethodStats methodStats : listener.getTestMethodStats()) {
259                 List<TestMethodStats> currentMethodStats =
260                         mergedTestHistoryResult.get(methodStats.getTestClassMethodName());
261                 if (currentMethodStats == null) {
262                     currentMethodStats = new ArrayList<>();
263                     currentMethodStats.add(methodStats);
264                     mergedTestHistoryResult.put(methodStats.getTestClassMethodName(), currentMethodStats);
265                 } else {
266                     currentMethodStats.add(methodStats);
267                 }
268             }
269         }
270 
271         // Update globalStatistics by iterating through mergedTestHistoryResult
272         int completedCount = 0, skipped = 0;
273         Map<String, List<TestMethodStats>> beforeAllFailures = new HashMap<>();
274 
275         for (Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet()) {
276             List<TestMethodStats> testMethodStats = entry.getValue();
277             String testClassMethodName = entry.getKey();
278 
279             // Handle @BeforeAll failures (null, empty, or ends with ".null" method names)
280             // But only if they actually failed (ERROR or FAILURE), not if they were skipped
281             if ((StringUtils.isBlank(testClassMethodName) || testClassMethodName.endsWith(".null"))
282                     && (testClassMethodName == null || !testClassMethodName.contains("$"))) {
283 
284                 // Check if this is actually a failure/error (not skipped or success)
285                 boolean isActualFailure = testMethodStats.stream()
286                         .anyMatch(stat -> stat.getResultType() == ReportEntryType.ERROR
287                                 || stat.getResultType() == ReportEntryType.FAILURE);
288 
289                 if (isActualFailure) {
290                     // Extract class name from the test class method name
291                     String className = extractClassNameFromMethodName(testClassMethodName);
292                     if (className != null) {
293                         if (beforeAllFailures.containsKey(className)) {
294                             List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
295                             previousMethodStats.addAll(testMethodStats);
296                             beforeAllFailures.put(className, previousMethodStats);
297                         } else {
298                             beforeAllFailures.put(className, new ArrayList<>(testMethodStats));
299                         }
300                     }
301                     // Skip normal processing of @BeforeAll failures because it needs special care
302                     continue;
303                 }
304                 // If it's skipped or success with null method name, fall through to normal processing
305             }
306 
307             completedCount++;
308 
309             List<ReportEntryType> resultTypes = new ArrayList<>();
310             for (TestMethodStats methodStats : testMethodStats) {
311                 resultTypes.add(methodStats.getResultType());
312             }
313 
314             switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) {
315                 case SUCCESS:
316                     // If there are multiple successful runs of the same test, count all of them
317                     int successCount = 0;
318                     for (ReportEntryType type : resultTypes) {
319                         if (type == ReportEntryType.SUCCESS) {
320                             successCount++;
321                         }
322                     }
323                     completedCount += successCount - 1;
324                     successTests.put(testClassMethodName, testMethodStats);
325                     break;
326                 case SKIPPED:
327                     skipped++;
328                     break;
329                 case FLAKE:
330                     flakyTests.put(testClassMethodName, testMethodStats);
331                     break;
332                 case FAILURE:
333                     failedTests.put(testClassMethodName, testMethodStats);
334                     break;
335                 case ERROR:
336                     errorTests.put(testClassMethodName, testMethodStats);
337                     break;
338                 default:
339                     throw new IllegalStateException("Get unknown test result type");
340             }
341         }
342 
343         // Loop over all success tests and find those that are passed flakes for beforeAll failures
344         for (Map.Entry<String, List<TestMethodStats>> entry : successTests.entrySet()) {
345             List<TestMethodStats> testMethodStats = entry.getValue();
346             String testClassMethodName = entry.getKey();
347             // If current test belong to class that failed during beforeAll store that info to proper log info
348             String className = extractClassNameFromMethodName(testClassMethodName);
349             if (beforeAllFailures.containsKey(className)) {
350                 List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
351                 previousMethodStats.addAll(testMethodStats);
352                 beforeAllFailures.put(className, previousMethodStats);
353             }
354         }
355 
356         // Process @BeforeAll failures after we know which classes have successful tests
357         for (Map.Entry<String, List<TestMethodStats>> entry : beforeAllFailures.entrySet()) {
358             String className = entry.getKey();
359             List<TestMethodStats> testMethodStats = entry.getValue();
360             String classNameKey = className + ".<beforeAll>";
361 
362             if (reportConfiguration.getRerunFailingTestsCount() > 0
363                     && testMethodStats.stream()
364                             .anyMatch(methodStats -> methodStats.getTestClassMethodName() != null
365                                     && !methodStats.getTestClassMethodName().isEmpty()
366                                     && methodStats.getResultType().equals(ReportEntryType.SUCCESS))) {
367                 flakyTests.put(classNameKey, testMethodStats);
368             } else {
369                 errorTests.put(classNameKey, testMethodStats);
370                 completedCount++;
371             }
372         }
373 
374         globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size());
375     }
376 
377     /**
378      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
379      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test.
380      *
381      * @param type   the type of results to be printed, could be error, failure or flake
382      * @return {@code true} if printed some lines
383      */
384     // Use default visibility for testing
385     boolean printTestFailures(TestResultType type) {
386         final Map<String, List<TestMethodStats>> testStats;
387         final Level level;
388         switch (type) {
389             case FAILURE:
390                 testStats = failedTests;
391                 level = Level.FAILURE;
392                 break;
393             case ERROR:
394                 testStats = errorTests;
395                 level = Level.FAILURE;
396                 break;
397             case FLAKE:
398                 testStats = flakyTests;
399                 level = Level.UNSTABLE;
400                 break;
401             default:
402                 return false;
403         }
404 
405         boolean printed = false;
406         if (!testStats.isEmpty()) {
407             log(type.getLogPrefix(), level);
408             printed = true;
409         }
410 
411         for (Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet()) {
412             List<TestMethodStats> testMethodStats = entry.getValue();
413             if (testMethodStats.size() == 1) {
414                 // No rerun, follow the original output format
415                 failure("  " + testMethodStats.get(0).getStackTraceWriter().smartTrimmedStackTrace());
416             } else {
417                 log(entry.getKey(), level);
418                 for (int i = 0; i < testMethodStats.size(); i++) {
419                     StackTraceWriter failureStackTrace = testMethodStats.get(i).getStackTraceWriter();
420                     if (failureStackTrace == null) {
421                         success("  Run " + (i + 1) + ": PASS");
422                     } else {
423                         failure("  Run " + (i + 1) + ": " + failureStackTrace.smartTrimmedStackTrace());
424                     }
425                 }
426                 log("");
427             }
428         }
429         return printed;
430     }
431 
432     /**
433      * Extract class name from test class method name like "com.example.TestClass.methodName"
434      */
435     private static String extractClassNameFromMethodName(String testClassMethodName) {
436         if (StringUtils.isBlank(testClassMethodName)) {
437             return null;
438         }
439         int lastDotIndex = testClassMethodName.lastIndexOf('.');
440         if (lastDotIndex > 0) {
441             return testClassMethodName.substring(0, lastDotIndex);
442         }
443         return null;
444     }
445 
446     /**
447      * Extract class name from stack trace when method name is null (e.g., @BeforeAll failures)
448      */
449     private static String extractClassNameFromStackTrace(TestMethodStats stats) {
450         if (stats.getStackTraceWriter() == null) {
451             return null;
452         }
453         String stackTrace = stats.getStackTraceWriter().smartTrimmedStackTrace();
454         if (stackTrace == null || stackTrace.isEmpty()) {
455             return null;
456         }
457 
458         // Strip everything after the first whitespace character
459         int firstWhitespace = stackTrace.indexOf(' ');
460         if (firstWhitespace > 0) {
461             stackTrace = stackTrace.substring(0, firstWhitespace);
462         }
463 
464         return extractClassNameFromMethodName(stackTrace);
465     }
466 
467     // Describe the result of a given test
468     enum TestResultType {
469         ERROR("Errors: "),
470         FAILURE("Failures: "),
471         FLAKE("Flakes: "),
472         SUCCESS("Success: "),
473         SKIPPED("Skipped: "),
474         UNKNOWN("Unknown: ");
475 
476         private final String logPrefix;
477 
478         TestResultType(String logPrefix) {
479             this.logPrefix = logPrefix;
480         }
481 
482         public String getLogPrefix() {
483             return logPrefix;
484         }
485     }
486 
487     private void log(String s, boolean success, boolean failures, boolean errors, boolean skipped, boolean flakes) {
488         Level level = resolveLevel(success, failures, errors, skipped, flakes);
489         log(s, level);
490     }
491 
492     private void log(String s, Level level) {
493         switch (level) {
494             case FAILURE:
495                 failure(s);
496                 break;
497             case UNSTABLE:
498                 warning(s);
499                 break;
500             case SUCCESS:
501                 success(s);
502                 break;
503             default:
504                 info(s);
505         }
506     }
507 
508     private void log(String s) {
509         consoleLogger.info(s);
510     }
511 
512     private void info(String s) {
513         MessageBuilder builder = buffer();
514         consoleLogger.info(builder.a(s).toString());
515     }
516 
517     private void warning(String s) {
518         MessageBuilder builder = buffer();
519         consoleLogger.warning(builder.warning(s).toString());
520     }
521 
522     private void success(String s) {
523         MessageBuilder builder = buffer();
524         consoleLogger.info(builder.success(s).toString());
525     }
526 
527     private void failure(String s) {
528         MessageBuilder builder = buffer();
529         consoleLogger.error(builder.failure(s).toString());
530     }
531 }