View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.plugin.surefire.report;
20  
21  import java.io.File;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.TreeMap;
28  import java.util.concurrent.ConcurrentLinkedQueue;
29  
30  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
31  import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
32  import org.apache.maven.plugin.surefire.log.api.Level;
33  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
34  import org.apache.maven.surefire.api.report.ReporterFactory;
35  import org.apache.maven.surefire.api.report.StackTraceWriter;
36  import org.apache.maven.surefire.api.report.TestOutputReportEntry;
37  import org.apache.maven.surefire.api.report.TestReportListener;
38  import org.apache.maven.surefire.api.suite.RunResult;
39  import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener;
40  import org.apache.maven.surefire.extensions.StatelessReportEventListener;
41  import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
42  import org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener;
43  import org.apache.maven.surefire.report.RunStatistics;
44  import org.apache.maven.surefire.shared.utils.logging.MessageBuilder;
45  
46  import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel;
47  import static org.apache.maven.plugin.surefire.report.ConsoleReporter.PLAIN;
48  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.ERROR;
49  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FAILURE;
50  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FLAKE;
51  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SKIPPED;
52  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SUCCESS;
53  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.UNKNOWN;
54  import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
55  import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
56  import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
57  import static org.apache.maven.surefire.api.util.internal.ObjectUtils.useNonNull;
58  import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer;
59  
60  /**
61   * Provides reporting modules on the plugin side.
62   * <br>
63   * Keeps a centralized count of test run results.
64   *
65   * @author Kristian Rosenvold
66   */
67  public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
68      private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<>();
69      private final StartupReportConfiguration reportConfiguration;
70      private final ConsoleLogger consoleLogger;
71      private final Integer forkNumber;
72  
73      private RunStatistics globalStats = new RunStatistics();
74  
75      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
76      private Map<String, List<TestMethodStats>> flakyTests;
77  
78      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
79      private Map<String, List<TestMethodStats>> failedTests;
80  
81      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
82      private Map<String, List<TestMethodStats>> errorTests;
83  
84      public DefaultReporterFactory(StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger) {
85          this(reportConfiguration, consoleLogger, null);
86      }
87  
88      public DefaultReporterFactory(
89              StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger, Integer forkNumber) {
90          this.reportConfiguration = reportConfiguration;
91          this.consoleLogger = consoleLogger;
92          this.forkNumber = forkNumber;
93      }
94  
95      @Override
96      public TestReportListener<TestOutputReportEntry> createTestReportListener() {
97          TestSetRunListener testSetRunListener = new TestSetRunListener(
98                  createConsoleReporter(),
99                  createFileReporter(),
100                 createSimpleXMLReporter(),
101                 createConsoleOutputReceiver(),
102                 createStatisticsReporter(),
103                 reportConfiguration.isTrimStackTrace(),
104                 PLAIN.equals(reportConfiguration.getReportFormat()),
105                 reportConfiguration.isBriefOrPlainFormat(),
106                 consoleLogger,
107                 reportConfiguration.getReporterFactoryOptions().isStatPerSourceName());
108         addListener(testSetRunListener);
109         return testSetRunListener;
110     }
111 
112     @Override
113     public File getReportsDirectory() {
114         return reportConfiguration.getReportsDirectory();
115     }
116 
117     private StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> createConsoleReporter() {
118         StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> consoleReporter =
119                 reportConfiguration.instantiateConsoleReporter(consoleLogger);
120         return useNonNull(consoleReporter, NullConsoleReporter.INSTANCE);
121     }
122 
123     private StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> createFileReporter() {
124         StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> fileReporter =
125                 reportConfiguration.instantiateFileReporter(forkNumber);
126         return useNonNull(fileReporter, NullFileReporter.INSTANCE);
127     }
128 
129     private StatelessReportEventListener<WrappedReportEntry, TestSetStats> createSimpleXMLReporter() {
130         StatelessReportEventListener<WrappedReportEntry, TestSetStats> xmlReporter =
131                 reportConfiguration.instantiateStatelessXmlReporter(forkNumber);
132         return useNonNull(xmlReporter, NullStatelessXmlReporter.INSTANCE);
133     }
134 
135     private ConsoleOutputReportEventListener createConsoleOutputReceiver() {
136         ConsoleOutputReportEventListener outputReporter =
137                 reportConfiguration.instantiateConsoleOutputFileReporter(forkNumber);
138         return useNonNull(outputReporter, NullConsoleOutputReceiver.INSTANCE);
139     }
140 
141     private StatisticsReporter createStatisticsReporter() {
142         StatisticsReporter statisticsReporter = reportConfiguration.getStatisticsReporter();
143         return useNonNull(statisticsReporter, NullStatisticsReporter.INSTANCE);
144     }
145 
146     @Override
147     public void mergeFromOtherFactories(Collection<DefaultReporterFactory> factories) {
148         for (DefaultReporterFactory factory : factories) {
149             listeners.addAll(factory.listeners);
150         }
151     }
152 
153     final void addListener(TestSetRunListener listener) {
154         listeners.add(listener);
155     }
156 
157     @Override
158     public RunResult close() {
159         mergeTestHistoryResult();
160         runCompleted();
161         for (TestSetRunListener listener : listeners) {
162             listener.close();
163         }
164         return globalStats.getRunResult();
165     }
166 
167     @Override
168     public void runStarting() {
169         if (reportConfiguration.isPrintSummary()) {
170             log("");
171             log("-------------------------------------------------------");
172             log(" T E S T S");
173             log("-------------------------------------------------------");
174         }
175     }
176 
177     private void runCompleted() {
178         if (reportConfiguration.isPrintSummary()) {
179             log("");
180             log("Results:");
181             log("");
182         }
183         boolean printedFailures = printTestFailures(TestResultType.FAILURE);
184         boolean printedErrors = printTestFailures(TestResultType.ERROR);
185         boolean printedFlakes = printTestFailures(TestResultType.FLAKE);
186         if (reportConfiguration.isPrintSummary()) {
187             if (printedFailures | printedErrors | printedFlakes) {
188                 log("");
189             }
190             boolean hasSuccessful = globalStats.getCompletedCount() > 0;
191             boolean hasSkipped = globalStats.getSkipped() > 0;
192             log(globalStats.getSummary(), hasSuccessful, printedFailures, printedErrors, hasSkipped, printedFlakes);
193             log("");
194         }
195     }
196 
197     public RunStatistics getGlobalRunStatistics() {
198         mergeTestHistoryResult();
199         return globalStats;
200     }
201 
202     /**
203      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
204      * if it only has errors or failures, then count its result based on its first run
205      *
206      * @param reportEntries the list of test run report type for a given test
207      * @param rerunFailingTestsCount configured rerun count for failing tests
208      * @return the type of test result
209      */
210     // Use default visibility for testing
211     static TestResultType getTestResultType(List<ReportEntryType> reportEntries, int rerunFailingTestsCount) {
212         if (reportEntries == null || reportEntries.isEmpty()) {
213             return UNKNOWN;
214         }
215 
216         boolean seenSuccess = false, seenFailure = false, seenError = false;
217         for (ReportEntryType resultType : reportEntries) {
218             if (resultType == ReportEntryType.SUCCESS) {
219                 seenSuccess = true;
220             } else if (resultType == ReportEntryType.FAILURE) {
221                 seenFailure = true;
222             } else if (resultType == ReportEntryType.ERROR) {
223                 seenError = true;
224             }
225         }
226 
227         if (seenFailure || seenError) {
228             if (seenSuccess && rerunFailingTestsCount > 0) {
229                 return TestResultType.FLAKE;
230             } else {
231                 return seenError ? TestResultType.ERROR : TestResultType.FAILURE;
232             }
233         } else if (seenSuccess) {
234             return TestResultType.SUCCESS;
235         } else {
236             return SKIPPED;
237         }
238     }
239 
240     /**
241      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
242      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
243      */
244     private void mergeTestHistoryResult() {
245         globalStats = new RunStatistics();
246         flakyTests = new TreeMap<>();
247         failedTests = new TreeMap<>();
248         errorTests = new TreeMap<>();
249 
250         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<>();
251         // Merge all the stats for tests from listeners
252         for (TestSetRunListener listener : listeners) {
253             for (TestMethodStats methodStats : listener.getTestMethodStats()) {
254                 List<TestMethodStats> currentMethodStats =
255                         mergedTestHistoryResult.get(methodStats.getTestClassMethodName());
256                 if (currentMethodStats == null) {
257                     currentMethodStats = new ArrayList<>();
258                     currentMethodStats.add(methodStats);
259                     mergedTestHistoryResult.put(methodStats.getTestClassMethodName(), currentMethodStats);
260                 } else {
261                     currentMethodStats.add(methodStats);
262                 }
263             }
264         }
265 
266         // Update globalStatistics by iterating through mergedTestHistoryResult
267         int completedCount = 0, skipped = 0;
268 
269         for (Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet()) {
270             List<TestMethodStats> testMethodStats = entry.getValue();
271             String testClassMethodName = entry.getKey();
272             completedCount++;
273 
274             List<ReportEntryType> resultTypes = new ArrayList<>();
275             for (TestMethodStats methodStats : testMethodStats) {
276                 resultTypes.add(methodStats.getResultType());
277             }
278 
279             switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) {
280                 case SUCCESS:
281                     // If there are multiple successful runs of the same test, count all of them
282                     int successCount = 0;
283                     for (ReportEntryType type : resultTypes) {
284                         if (type == ReportEntryType.SUCCESS) {
285                             successCount++;
286                         }
287                     }
288                     completedCount += successCount - 1;
289                     break;
290                 case SKIPPED:
291                     skipped++;
292                     break;
293                 case FLAKE:
294                     flakyTests.put(testClassMethodName, testMethodStats);
295                     break;
296                 case FAILURE:
297                     failedTests.put(testClassMethodName, testMethodStats);
298                     break;
299                 case ERROR:
300                     errorTests.put(testClassMethodName, testMethodStats);
301                     break;
302                 default:
303                     throw new IllegalStateException("Get unknown test result type");
304             }
305         }
306 
307         globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size());
308     }
309 
310     /**
311      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
312      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
313      *
314      * @param type   the type of results to be printed, could be error, failure or flake
315      * @return {@code true} if printed some lines
316      */
317     // Use default visibility for testing
318     boolean printTestFailures(TestResultType type) {
319         final Map<String, List<TestMethodStats>> testStats;
320         final Level level;
321         switch (type) {
322             case FAILURE:
323                 testStats = failedTests;
324                 level = Level.FAILURE;
325                 break;
326             case ERROR:
327                 testStats = errorTests;
328                 level = Level.FAILURE;
329                 break;
330             case FLAKE:
331                 testStats = flakyTests;
332                 level = Level.UNSTABLE;
333                 break;
334             default:
335                 return false;
336         }
337 
338         boolean printed = false;
339         if (!testStats.isEmpty()) {
340             log(type.getLogPrefix(), level);
341             printed = true;
342         }
343 
344         for (Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet()) {
345             List<TestMethodStats> testMethodStats = entry.getValue();
346             if (testMethodStats.size() == 1) {
347                 // No rerun, follow the original output format
348                 failure("  " + testMethodStats.get(0).getStackTraceWriter().smartTrimmedStackTrace());
349             } else {
350                 log(entry.getKey(), level);
351                 for (int i = 0; i < testMethodStats.size(); i++) {
352                     StackTraceWriter failureStackTrace = testMethodStats.get(i).getStackTraceWriter();
353                     if (failureStackTrace == null) {
354                         success("  Run " + (i + 1) + ": PASS");
355                     } else {
356                         failure("  Run " + (i + 1) + ": " + failureStackTrace.smartTrimmedStackTrace());
357                     }
358                 }
359                 log("");
360             }
361         }
362         return printed;
363     }
364 
365     // Describe the result of a given test
366     enum TestResultType {
367         ERROR("Errors: "),
368         FAILURE("Failures: "),
369         FLAKE("Flakes: "),
370         SUCCESS("Success: "),
371         SKIPPED("Skipped: "),
372         UNKNOWN("Unknown: ");
373 
374         private final String logPrefix;
375 
376         TestResultType(String logPrefix) {
377             this.logPrefix = logPrefix;
378         }
379 
380         public String getLogPrefix() {
381             return logPrefix;
382         }
383     }
384 
385     private void log(String s, boolean success, boolean failures, boolean errors, boolean skipped, boolean flakes) {
386         Level level = resolveLevel(success, failures, errors, skipped, flakes);
387         log(s, level);
388     }
389 
390     private void log(String s, Level level) {
391         switch (level) {
392             case FAILURE:
393                 failure(s);
394                 break;
395             case UNSTABLE:
396                 warning(s);
397                 break;
398             case SUCCESS:
399                 success(s);
400                 break;
401             default:
402                 info(s);
403         }
404     }
405 
406     private void log(String s) {
407         consoleLogger.info(s);
408     }
409 
410     private void info(String s) {
411         MessageBuilder builder = buffer();
412         consoleLogger.info(builder.a(s).toString());
413     }
414 
415     private void warning(String s) {
416         MessageBuilder builder = buffer();
417         consoleLogger.warning(builder.warning(s).toString());
418     }
419 
420     private void success(String s) {
421         MessageBuilder builder = buffer();
422         consoleLogger.info(builder.success(s).toString());
423     }
424 
425     private void failure(String s) {
426         MessageBuilder builder = buffer();
427         consoleLogger.error(builder.failure(s).toString());
428     }
429 }