View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.plugin.surefire.report;
20  
21  import java.io.File;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.TreeMap;
28  import java.util.concurrent.ConcurrentLinkedQueue;
29  
30  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
31  import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
32  import org.apache.maven.plugin.surefire.log.api.Level;
33  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
34  import org.apache.maven.surefire.api.report.ReporterFactory;
35  import org.apache.maven.surefire.api.report.StackTraceWriter;
36  import org.apache.maven.surefire.api.report.TestOutputReportEntry;
37  import org.apache.maven.surefire.api.report.TestReportListener;
38  import org.apache.maven.surefire.api.suite.RunResult;
39  import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener;
40  import org.apache.maven.surefire.extensions.StatelessReportEventListener;
41  import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
42  import org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener;
43  import org.apache.maven.surefire.report.RunStatistics;
44  import org.apache.maven.surefire.shared.utils.logging.MessageBuilder;
45  
46  import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel;
47  import static org.apache.maven.plugin.surefire.report.ConsoleReporter.PLAIN;
48  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.ERROR;
49  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FAILURE;
50  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.FLAKE;
51  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SKIPPED;
52  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.SUCCESS;
53  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.UNKNOWN;
54  import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
55  import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
56  import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
57  import static org.apache.maven.surefire.api.util.internal.ObjectUtils.useNonNull;
58  import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer;
59  
60  /**
61   * Provides reporting modules on the plugin side.
62   * <br>
63   * Keeps a centralized count of test run results.
64   *
65   * @author Kristian Rosenvold
66   */
67  public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
68      private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<>();
69      private final StartupReportConfiguration reportConfiguration;
70      private final ConsoleLogger consoleLogger;
71      private final Integer forkNumber;
72  
73      private RunStatistics globalStats = new RunStatistics();
74  
75      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
76      private Map<String, List<TestMethodStats>> flakyTests;
77  
78      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
79      private Map<String, List<TestMethodStats>> failedTests;
80  
81      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
82      private Map<String, List<TestMethodStats>> errorTests;
83  
84      public DefaultReporterFactory(StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger) {
85          this(reportConfiguration, consoleLogger, null);
86      }
87  
88      public DefaultReporterFactory(
89              StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger, Integer forkNumber) {
90          this.reportConfiguration = reportConfiguration;
91          this.consoleLogger = consoleLogger;
92          this.forkNumber = forkNumber;
93      }
94  
95      @Override
96      public TestReportListener<TestOutputReportEntry> createTestReportListener() {
97          TestSetRunListener testSetRunListener = new TestSetRunListener(
98                  createConsoleReporter(),
99                  createFileReporter(),
100                 createSimpleXMLReporter(),
101                 createConsoleOutputReceiver(),
102                 createStatisticsReporter(),
103                 reportConfiguration.isTrimStackTrace(),
104                 PLAIN.equals(reportConfiguration.getReportFormat()),
105                 reportConfiguration.isBriefOrPlainFormat(),
106                 consoleLogger);
107         addListener(testSetRunListener);
108         return testSetRunListener;
109     }
110 
111     @Override
112     public File getReportsDirectory() {
113         return reportConfiguration.getReportsDirectory();
114     }
115 
116     private StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> createConsoleReporter() {
117         StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> consoleReporter =
118                 reportConfiguration.instantiateConsoleReporter(consoleLogger);
119         return useNonNull(consoleReporter, NullConsoleReporter.INSTANCE);
120     }
121 
122     private StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> createFileReporter() {
123         StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> fileReporter =
124                 reportConfiguration.instantiateFileReporter(forkNumber);
125         return useNonNull(fileReporter, NullFileReporter.INSTANCE);
126     }
127 
128     private StatelessReportEventListener<WrappedReportEntry, TestSetStats> createSimpleXMLReporter() {
129         StatelessReportEventListener<WrappedReportEntry, TestSetStats> xmlReporter =
130                 reportConfiguration.instantiateStatelessXmlReporter(forkNumber);
131         return useNonNull(xmlReporter, NullStatelessXmlReporter.INSTANCE);
132     }
133 
134     private ConsoleOutputReportEventListener createConsoleOutputReceiver() {
135         ConsoleOutputReportEventListener outputReporter =
136                 reportConfiguration.instantiateConsoleOutputFileReporter(forkNumber);
137         return useNonNull(outputReporter, NullConsoleOutputReceiver.INSTANCE);
138     }
139 
140     private StatisticsReporter createStatisticsReporter() {
141         StatisticsReporter statisticsReporter = reportConfiguration.getStatisticsReporter();
142         return useNonNull(statisticsReporter, NullStatisticsReporter.INSTANCE);
143     }
144 
145     @Override
146     public void mergeFromOtherFactories(Collection<DefaultReporterFactory> factories) {
147         for (DefaultReporterFactory factory : factories) {
148             listeners.addAll(factory.listeners);
149         }
150     }
151 
152     final void addListener(TestSetRunListener listener) {
153         listeners.add(listener);
154     }
155 
156     @Override
157     public RunResult close() {
158         mergeTestHistoryResult();
159         runCompleted();
160         for (TestSetRunListener listener : listeners) {
161             listener.close();
162         }
163         return globalStats.getRunResult();
164     }
165 
166     @Override
167     public void runStarting() {
168         if (reportConfiguration.isPrintSummary()) {
169             log("");
170             log("-------------------------------------------------------");
171             log(" T E S T S");
172             log("-------------------------------------------------------");
173         }
174     }
175 
176     private void runCompleted() {
177         if (reportConfiguration.isPrintSummary()) {
178             log("");
179             log("Results:");
180             log("");
181         }
182         boolean printedFailures = printTestFailures(TestResultType.FAILURE);
183         boolean printedErrors = printTestFailures(TestResultType.ERROR);
184         boolean printedFlakes = printTestFailures(TestResultType.FLAKE);
185         if (reportConfiguration.isPrintSummary()) {
186             if (printedFailures | printedErrors | printedFlakes) {
187                 log("");
188             }
189             boolean hasSuccessful = globalStats.getCompletedCount() > 0;
190             boolean hasSkipped = globalStats.getSkipped() > 0;
191             log(globalStats.getSummary(), hasSuccessful, printedFailures, printedErrors, hasSkipped, printedFlakes);
192             log("");
193         }
194     }
195 
196     public RunStatistics getGlobalRunStatistics() {
197         mergeTestHistoryResult();
198         return globalStats;
199     }
200 
201     /**
202      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
203      * if it only has errors or failures, then count its result based on its first run
204      *
205      * @param reportEntries the list of test run report type for a given test
206      * @param rerunFailingTestsCount configured rerun count for failing tests
207      * @return the type of test result
208      */
209     // Use default visibility for testing
210     static TestResultType getTestResultType(List<ReportEntryType> reportEntries, int rerunFailingTestsCount) {
211         if (reportEntries == null || reportEntries.isEmpty()) {
212             return UNKNOWN;
213         }
214 
215         boolean seenSuccess = false, seenFailure = false, seenError = false;
216         for (ReportEntryType resultType : reportEntries) {
217             if (resultType == ReportEntryType.SUCCESS) {
218                 seenSuccess = true;
219             } else if (resultType == ReportEntryType.FAILURE) {
220                 seenFailure = true;
221             } else if (resultType == ReportEntryType.ERROR) {
222                 seenError = true;
223             }
224         }
225 
226         if (seenFailure || seenError) {
227             if (seenSuccess && rerunFailingTestsCount > 0) {
228                 return TestResultType.FLAKE;
229             } else {
230                 return seenError ? TestResultType.ERROR : TestResultType.FAILURE;
231             }
232         } else if (seenSuccess) {
233             return TestResultType.SUCCESS;
234         } else {
235             return SKIPPED;
236         }
237     }
238 
239     /**
240      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
241      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
242      */
243     private void mergeTestHistoryResult() {
244         globalStats = new RunStatistics();
245         flakyTests = new TreeMap<>();
246         failedTests = new TreeMap<>();
247         errorTests = new TreeMap<>();
248 
249         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<>();
250         // Merge all the stats for tests from listeners
251         for (TestSetRunListener listener : listeners) {
252             for (TestMethodStats methodStats : listener.getTestMethodStats()) {
253                 List<TestMethodStats> currentMethodStats =
254                         mergedTestHistoryResult.get(methodStats.getTestClassMethodName());
255                 if (currentMethodStats == null) {
256                     currentMethodStats = new ArrayList<>();
257                     currentMethodStats.add(methodStats);
258                     mergedTestHistoryResult.put(methodStats.getTestClassMethodName(), currentMethodStats);
259                 } else {
260                     currentMethodStats.add(methodStats);
261                 }
262             }
263         }
264 
265         // Update globalStatistics by iterating through mergedTestHistoryResult
266         int completedCount = 0, skipped = 0;
267 
268         for (Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet()) {
269             List<TestMethodStats> testMethodStats = entry.getValue();
270             String testClassMethodName = entry.getKey();
271             completedCount++;
272 
273             List<ReportEntryType> resultTypes = new ArrayList<>();
274             for (TestMethodStats methodStats : testMethodStats) {
275                 resultTypes.add(methodStats.getResultType());
276             }
277 
278             switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) {
279                 case SUCCESS:
280                     // If there are multiple successful runs of the same test, count all of them
281                     int successCount = 0;
282                     for (ReportEntryType type : resultTypes) {
283                         if (type == ReportEntryType.SUCCESS) {
284                             successCount++;
285                         }
286                     }
287                     completedCount += successCount - 1;
288                     break;
289                 case SKIPPED:
290                     skipped++;
291                     break;
292                 case FLAKE:
293                     flakyTests.put(testClassMethodName, testMethodStats);
294                     break;
295                 case FAILURE:
296                     failedTests.put(testClassMethodName, testMethodStats);
297                     break;
298                 case ERROR:
299                     errorTests.put(testClassMethodName, testMethodStats);
300                     break;
301                 default:
302                     throw new IllegalStateException("Get unknown test result type");
303             }
304         }
305 
306         globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size());
307     }
308 
309     /**
310      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
311      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
312      *
313      * @param type   the type of results to be printed, could be error, failure or flake
314      * @return {@code true} if printed some lines
315      */
316     // Use default visibility for testing
317     boolean printTestFailures(TestResultType type) {
318         final Map<String, List<TestMethodStats>> testStats;
319         final Level level;
320         switch (type) {
321             case FAILURE:
322                 testStats = failedTests;
323                 level = Level.FAILURE;
324                 break;
325             case ERROR:
326                 testStats = errorTests;
327                 level = Level.FAILURE;
328                 break;
329             case FLAKE:
330                 testStats = flakyTests;
331                 level = Level.UNSTABLE;
332                 break;
333             default:
334                 return false;
335         }
336 
337         boolean printed = false;
338         if (!testStats.isEmpty()) {
339             log(type.getLogPrefix(), level);
340             printed = true;
341         }
342 
343         for (Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet()) {
344             List<TestMethodStats> testMethodStats = entry.getValue();
345             if (testMethodStats.size() == 1) {
346                 // No rerun, follow the original output format
347                 failure("  " + testMethodStats.get(0).getStackTraceWriter().smartTrimmedStackTrace());
348             } else {
349                 log(entry.getKey(), level);
350                 for (int i = 0; i < testMethodStats.size(); i++) {
351                     StackTraceWriter failureStackTrace = testMethodStats.get(i).getStackTraceWriter();
352                     if (failureStackTrace == null) {
353                         success("  Run " + (i + 1) + ": PASS");
354                     } else {
355                         failure("  Run " + (i + 1) + ": " + failureStackTrace.smartTrimmedStackTrace());
356                     }
357                 }
358                 log("");
359             }
360         }
361         return printed;
362     }
363 
364     // Describe the result of a given test
365     enum TestResultType {
366         ERROR("Errors: "),
367         FAILURE("Failures: "),
368         FLAKE("Flakes: "),
369         SUCCESS("Success: "),
370         SKIPPED("Skipped: "),
371         UNKNOWN("Unknown: ");
372 
373         private final String logPrefix;
374 
375         TestResultType(String logPrefix) {
376             this.logPrefix = logPrefix;
377         }
378 
379         public String getLogPrefix() {
380             return logPrefix;
381         }
382     }
383 
384     private void log(String s, boolean success, boolean failures, boolean errors, boolean skipped, boolean flakes) {
385         Level level = resolveLevel(success, failures, errors, skipped, flakes);
386         log(s, level);
387     }
388 
389     private void log(String s, Level level) {
390         switch (level) {
391             case FAILURE:
392                 failure(s);
393                 break;
394             case UNSTABLE:
395                 warning(s);
396                 break;
397             case SUCCESS:
398                 success(s);
399                 break;
400             default:
401                 info(s);
402         }
403     }
404 
405     private void log(String s) {
406         consoleLogger.info(s);
407     }
408 
409     private void info(String s) {
410         MessageBuilder builder = buffer();
411         consoleLogger.info(builder.a(s).toString());
412     }
413 
414     private void warning(String s) {
415         MessageBuilder builder = buffer();
416         consoleLogger.warning(builder.warning(s).toString());
417     }
418 
419     private void success(String s) {
420         MessageBuilder builder = buffer();
421         consoleLogger.info(builder.success(s).toString());
422     }
423 
424     private void failure(String s) {
425         MessageBuilder builder = buffer();
426         consoleLogger.error(builder.failure(s).toString());
427     }
428 }