View Javadoc
1   package org.apache.maven.plugin.surefire.report;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
23  import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
24  import org.apache.maven.plugin.surefire.log.api.Level;
25  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
26  import org.apache.maven.surefire.api.report.TestOutputReportEntry;
27  import org.apache.maven.surefire.shared.utils.logging.MessageBuilder;
28  import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener;
29  import org.apache.maven.surefire.extensions.StatelessReportEventListener;
30  import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
31  import org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener;
32  import org.apache.maven.surefire.api.report.ReporterFactory;
33  import org.apache.maven.surefire.api.report.TestReportListener;
34  import org.apache.maven.surefire.report.RunStatistics;
35  import org.apache.maven.surefire.api.report.StackTraceWriter;
36  import org.apache.maven.surefire.api.suite.RunResult;
37  
38  import java.io.File;
39  import java.util.ArrayList;
40  import java.util.Collection;
41  import java.util.HashMap;
42  import java.util.List;
43  import java.util.Map;
44  import java.util.TreeMap;
45  import java.util.concurrent.ConcurrentLinkedQueue;
46  
47  import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel;
48  import static org.apache.maven.plugin.surefire.report.ConsoleReporter.PLAIN;
49  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.error;
50  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.failure;
51  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.flake;
52  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.skipped;
53  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.success;
54  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.unknown;
55  import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
56  import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
57  import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
58  import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer;
59  import static org.apache.maven.surefire.api.util.internal.ObjectUtils.useNonNull;
60  
61  /**
62   * Provides reporting modules on the plugin side.
63   * <br>
64   * Keeps a centralized count of test run results.
65   *
66   * @author Kristian Rosenvold
67   */
68  public class DefaultReporterFactory
69      implements ReporterFactory, ReportsMerger
70  {
71      private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<>();
72      private final StartupReportConfiguration reportConfiguration;
73      private final ConsoleLogger consoleLogger;
74      private final Integer forkNumber;
75  
76      private RunStatistics globalStats = new RunStatistics();
77  
78      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
79      private Map<String, List<TestMethodStats>> flakyTests;
80  
81      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
82      private Map<String, List<TestMethodStats>> failedTests;
83  
84      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
85      private Map<String, List<TestMethodStats>> errorTests;
86  
87      public DefaultReporterFactory( StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger )
88      {
89          this( reportConfiguration, consoleLogger, null );
90      }
91  
92      public DefaultReporterFactory( StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger,
93                                     Integer forkNumber )
94      {
95          this.reportConfiguration = reportConfiguration;
96          this.consoleLogger = consoleLogger;
97          this.forkNumber = forkNumber;
98      }
99  
100     @Override
101     public TestReportListener<TestOutputReportEntry> createTestReportListener()
102     {
103         TestSetRunListener testSetRunListener =
104             new TestSetRunListener( createConsoleReporter(),
105                                     createFileReporter(),
106                                     createSimpleXMLReporter(),
107                                     createConsoleOutputReceiver(),
108                                     createStatisticsReporter(),
109                                     reportConfiguration.isTrimStackTrace(),
110                                     PLAIN.equals( reportConfiguration.getReportFormat() ),
111                                     reportConfiguration.isBriefOrPlainFormat(),
112                                     consoleLogger );
113         addListener( testSetRunListener );
114         return testSetRunListener;
115     }
116 
117     @Override
118     public File getReportsDirectory()
119     {
120         return reportConfiguration.getReportsDirectory();
121     }
122 
123     private StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> createConsoleReporter()
124     {
125         StatelessTestsetInfoConsoleReportEventListener<WrappedReportEntry, TestSetStats> consoleReporter =
126                 reportConfiguration.instantiateConsoleReporter( consoleLogger );
127         return useNonNull( consoleReporter, NullConsoleReporter.INSTANCE );
128     }
129 
130     private StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> createFileReporter()
131     {
132         StatelessTestsetInfoFileReportEventListener<WrappedReportEntry, TestSetStats> fileReporter =
133                 reportConfiguration.instantiateFileReporter( forkNumber );
134         return useNonNull( fileReporter, NullFileReporter.INSTANCE );
135     }
136 
137     private StatelessReportEventListener<WrappedReportEntry, TestSetStats> createSimpleXMLReporter()
138     {
139         StatelessReportEventListener<WrappedReportEntry, TestSetStats> xmlReporter =
140                 reportConfiguration.instantiateStatelessXmlReporter( forkNumber );
141         return useNonNull( xmlReporter, NullStatelessXmlReporter.INSTANCE );
142     }
143 
144     private ConsoleOutputReportEventListener createConsoleOutputReceiver()
145     {
146         ConsoleOutputReportEventListener outputReporter =
147                 reportConfiguration.instantiateConsoleOutputFileReporter( forkNumber );
148         return useNonNull( outputReporter, NullConsoleOutputReceiver.INSTANCE );
149     }
150 
151     private StatisticsReporter createStatisticsReporter()
152     {
153         StatisticsReporter statisticsReporter = reportConfiguration.getStatisticsReporter();
154         return useNonNull( statisticsReporter, NullStatisticsReporter.INSTANCE );
155     }
156 
157     @Override
158     public void mergeFromOtherFactories( Collection<DefaultReporterFactory> factories )
159     {
160         for ( DefaultReporterFactory factory : factories )
161         {
162             listeners.addAll( factory.listeners );
163         }
164     }
165 
166     final void addListener( TestSetRunListener listener )
167     {
168         listeners.add( listener );
169     }
170 
171     @Override
172     public RunResult close()
173     {
174         mergeTestHistoryResult();
175         runCompleted();
176         for ( TestSetRunListener listener : listeners )
177         {
178             listener.close();
179         }
180         return globalStats.getRunResult();
181     }
182 
183     @Override
184     public void runStarting()
185     {
186         if ( reportConfiguration.isPrintSummary() )
187         {
188             log( "" );
189             log( "-------------------------------------------------------" );
190             log( " T E S T S" );
191             log( "-------------------------------------------------------" );
192         }
193     }
194 
195     private void runCompleted()
196     {
197         if ( reportConfiguration.isPrintSummary() )
198         {
199             log( "" );
200             log( "Results:" );
201             log( "" );
202         }
203         boolean printedFailures = printTestFailures( failure );
204         boolean printedErrors = printTestFailures( error );
205         boolean printedFlakes = printTestFailures( flake );
206         if ( reportConfiguration.isPrintSummary() )
207         {
208             if ( printedFailures | printedErrors | printedFlakes )
209             {
210                 log( "" );
211             }
212             boolean hasSuccessful = globalStats.getCompletedCount() > 0;
213             boolean hasSkipped = globalStats.getSkipped() > 0;
214             log( globalStats.getSummary(), hasSuccessful, printedFailures, printedErrors, hasSkipped, printedFlakes );
215             log( "" );
216         }
217     }
218 
219     public RunStatistics getGlobalRunStatistics()
220     {
221         mergeTestHistoryResult();
222         return globalStats;
223     }
224 
225     /**
226      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
227      * if it only has errors or failures, then count its result based on its first run
228      *
229      * @param reportEntries the list of test run report type for a given test
230      * @param rerunFailingTestsCount configured rerun count for failing tests
231      * @return the type of test result
232      */
233     // Use default visibility for testing
234     static TestResultType getTestResultType( List<ReportEntryType> reportEntries, int rerunFailingTestsCount  )
235     {
236         if ( reportEntries == null || reportEntries.isEmpty() )
237         {
238             return unknown;
239         }
240 
241         boolean seenSuccess = false, seenFailure = false, seenError = false;
242         for ( ReportEntryType resultType : reportEntries )
243         {
244             if ( resultType == SUCCESS )
245             {
246                 seenSuccess = true;
247             }
248             else if ( resultType == FAILURE )
249             {
250                 seenFailure = true;
251             }
252             else if ( resultType == ERROR )
253             {
254                 seenError = true;
255             }
256         }
257 
258         if ( seenFailure || seenError )
259         {
260             if ( seenSuccess && rerunFailingTestsCount > 0 )
261             {
262                 return flake;
263             }
264             else
265             {
266                 return seenError ? error : failure;
267             }
268         }
269         else if ( seenSuccess )
270         {
271             return success;
272         }
273         else
274         {
275             return skipped;
276         }
277     }
278 
279     /**
280      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
281      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
282      */
283     private void mergeTestHistoryResult()
284     {
285         globalStats = new RunStatistics();
286         flakyTests = new TreeMap<>();
287         failedTests = new TreeMap<>();
288         errorTests = new TreeMap<>();
289 
290         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<>();
291         // Merge all the stats for tests from listeners
292         for ( TestSetRunListener listener : listeners )
293         {
294             for ( TestMethodStats methodStats : listener.getTestMethodStats() )
295             {
296                 List<TestMethodStats> currentMethodStats =
297                     mergedTestHistoryResult.get( methodStats.getTestClassMethodName() );
298                 if ( currentMethodStats == null )
299                 {
300                     currentMethodStats = new ArrayList<>();
301                     currentMethodStats.add( methodStats );
302                     mergedTestHistoryResult.put( methodStats.getTestClassMethodName(), currentMethodStats );
303                 }
304                 else
305                 {
306                     currentMethodStats.add( methodStats );
307                 }
308             }
309         }
310 
311         // Update globalStatistics by iterating through mergedTestHistoryResult
312         int completedCount = 0, skipped = 0;
313 
314         for ( Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet() )
315         {
316             List<TestMethodStats> testMethodStats = entry.getValue();
317             String testClassMethodName = entry.getKey();
318             completedCount++;
319 
320             List<ReportEntryType> resultTypes = new ArrayList<>();
321             for ( TestMethodStats methodStats : testMethodStats )
322             {
323                 resultTypes.add( methodStats.getResultType() );
324             }
325 
326             switch ( getTestResultType( resultTypes, reportConfiguration.getRerunFailingTestsCount() ) )
327             {
328                 case success:
329                     // If there are multiple successful runs of the same test, count all of them
330                     int successCount = 0;
331                     for ( ReportEntryType type : resultTypes )
332                     {
333                         if ( type == SUCCESS )
334                         {
335                             successCount++;
336                         }
337                     }
338                     completedCount += successCount - 1;
339                     break;
340                 case skipped:
341                     skipped++;
342                     break;
343                 case flake:
344                     flakyTests.put( testClassMethodName, testMethodStats );
345                     break;
346                 case failure:
347                     failedTests.put( testClassMethodName, testMethodStats );
348                     break;
349                 case error:
350                     errorTests.put( testClassMethodName, testMethodStats );
351                     break;
352                 default:
353                     throw new IllegalStateException( "Get unknown test result type" );
354             }
355         }
356 
357         globalStats.set( completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size() );
358     }
359 
360     /**
361      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
362      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
363      *
364      * @param type   the type of results to be printed, could be error, failure or flake
365      * @return {@code true} if printed some lines
366      */
367     // Use default visibility for testing
368     boolean printTestFailures( TestResultType type )
369     {
370         final Map<String, List<TestMethodStats>> testStats;
371         final Level level;
372         switch ( type )
373         {
374             case failure:
375                 testStats = failedTests;
376                 level = Level.FAILURE;
377                 break;
378             case error:
379                 testStats = errorTests;
380                 level = Level.FAILURE;
381                 break;
382             case flake:
383                 testStats = flakyTests;
384                 level = Level.UNSTABLE;
385                 break;
386             default:
387                 return false;
388         }
389 
390         boolean printed = false;
391         if ( !testStats.isEmpty() )
392         {
393             log( type.getLogPrefix(), level );
394             printed = true;
395         }
396 
397         for ( Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet() )
398         {
399             List<TestMethodStats> testMethodStats = entry.getValue();
400             if ( testMethodStats.size() == 1 )
401             {
402                 // No rerun, follow the original output format
403                 failure( "  " + testMethodStats.get( 0 ).getStackTraceWriter().smartTrimmedStackTrace() );
404             }
405             else
406             {
407                 log( entry.getKey(), level );
408                 for ( int i = 0; i < testMethodStats.size(); i++ )
409                 {
410                     StackTraceWriter failureStackTrace = testMethodStats.get( i ).getStackTraceWriter();
411                     if ( failureStackTrace == null )
412                     {
413                         success( "  Run " + ( i + 1 ) + ": PASS" );
414                     }
415                     else
416                     {
417                         failure( "  Run " + ( i + 1 ) + ": " + failureStackTrace.smartTrimmedStackTrace() );
418                     }
419                 }
420                 log( "" );
421             }
422         }
423         return printed;
424     }
425 
426     // Describe the result of a given test
427     enum TestResultType
428     {
429 
430         error(   "Errors: "   ),
431         failure( "Failures: " ),
432         flake(   "Flakes: "   ),
433         success( "Success: "  ),
434         skipped( "Skipped: "  ),
435         unknown( "Unknown: "  );
436 
437         private final String logPrefix;
438 
439         TestResultType( String logPrefix )
440         {
441             this.logPrefix = logPrefix;
442         }
443 
444         public String getLogPrefix()
445         {
446             return logPrefix;
447         }
448     }
449 
450     private void log( String s, boolean success, boolean failures, boolean errors, boolean skipped, boolean flakes )
451     {
452         Level level = resolveLevel( success, failures, errors, skipped, flakes );
453         log( s, level );
454     }
455 
456     private void log( String s, Level level )
457     {
458         switch ( level )
459         {
460             case FAILURE:
461                 failure( s );
462                 break;
463             case UNSTABLE:
464                 warning( s );
465                 break;
466             case SUCCESS:
467                 success( s );
468                 break;
469             default:
470                 info( s );
471         }
472     }
473 
474     private void log( String s )
475     {
476         consoleLogger.info( s );
477     }
478 
479     private void info( String s )
480     {
481         MessageBuilder builder = buffer();
482         consoleLogger.info( builder.a( s ).toString() );
483     }
484 
485     private void warning( String s )
486     {
487         MessageBuilder builder = buffer();
488         consoleLogger.warning( builder.warning( s ).toString() );
489     }
490 
491     private void success( String s )
492     {
493         MessageBuilder builder = buffer();
494         consoleLogger.info( builder.success( s ).toString() );
495     }
496 
497     private void failure( String s )
498     {
499         MessageBuilder builder = buffer();
500         consoleLogger.error( builder.failure( s ).toString() );
501     }
502 }