View Javadoc
1   package org.apache.maven.plugin.surefire.report;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
23  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
24  import org.apache.maven.surefire.report.DefaultDirectConsoleReporter;
25  import org.apache.maven.surefire.report.ReporterFactory;
26  import org.apache.maven.surefire.report.RunListener;
27  import org.apache.maven.surefire.report.RunStatistics;
28  import org.apache.maven.surefire.report.StackTraceWriter;
29  import org.apache.maven.surefire.suite.RunResult;
30  
31  import java.util.ArrayList;
32  import java.util.Collection;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.TreeMap;
37  import java.util.concurrent.ConcurrentLinkedQueue;
38  
39  /**
40   * Provides reporting modules on the plugin side.
41   * <p/>
42   * Keeps a centralized count of test run results.
43   *
44   * @author Kristian Rosenvold
45   */
46  public class DefaultReporterFactory
47      implements ReporterFactory
48  {
49  
50      private RunStatistics globalStats = new RunStatistics();
51  
52      private final StartupReportConfiguration reportConfiguration;
53  
54      private final StatisticsReporter statisticsReporter;
55  
56      private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<TestSetRunListener>();
57  
58      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
59      private Map<String, List<TestMethodStats>> flakyTests;
60  
61      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
62      private Map<String, List<TestMethodStats>> failedTests;
63  
64      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
65      private Map<String, List<TestMethodStats>> errorTests;
66  
67      public DefaultReporterFactory( StartupReportConfiguration reportConfiguration )
68      {
69          this.reportConfiguration = reportConfiguration;
70          this.statisticsReporter = reportConfiguration.instantiateStatisticsReporter();
71      }
72  
73      public RunListener createReporter()
74      {
75          TestSetRunListener testSetRunListener =
76              new TestSetRunListener( reportConfiguration.instantiateConsoleReporter(),
77                                      reportConfiguration.instantiateFileReporter(),
78                                      reportConfiguration.instantiateStatelessXmlReporter(),
79                                      reportConfiguration.instantiateConsoleOutputFileReporter(), statisticsReporter,
80                                      reportConfiguration.isTrimStackTrace(),
81                                      ConsoleReporter.PLAIN.equals( reportConfiguration.getReportFormat() ),
82                                      reportConfiguration.isBriefOrPlainFormat() );
83          addListener( testSetRunListener );
84          return testSetRunListener;
85      }
86  
87      public void mergeFromOtherFactories( Collection<DefaultReporterFactory> factories )
88      {
89          for ( DefaultReporterFactory factory : factories )
90          {
91              for ( TestSetRunListener listener : factory.listeners )
92              {
93                  listeners.add( listener );
94              }
95          }
96      }
97  
98      final void addListener( TestSetRunListener listener )
99      {
100         listeners.add( listener );
101     }
102 
103     public RunResult close()
104     {
105         mergeTestHistoryResult();
106         runCompleted();
107         for ( TestSetRunListener listener : listeners )
108         {
109             listener.close();
110         }
111         return globalStats.getRunResult();
112     }
113 
114     private DefaultDirectConsoleReporter createConsoleLogger()
115     {
116         return new DefaultDirectConsoleReporter( reportConfiguration.getOriginalSystemOut() );
117     }
118 
119     public void runStarting()
120     {
121         final DefaultDirectConsoleReporter consoleReporter = createConsoleLogger();
122         consoleReporter.info( "" );
123         consoleReporter.info( "-------------------------------------------------------" );
124         consoleReporter.info( " T E S T S" );
125         consoleReporter.info( "-------------------------------------------------------" );
126     }
127 
128     private void runCompleted()
129     {
130         final DefaultDirectConsoleReporter logger = createConsoleLogger();
131         if ( reportConfiguration.isPrintSummary() )
132         {
133             logger.info( "" );
134             logger.info( "Results :" );
135             logger.info( "" );
136         }
137         boolean printedFailures = printTestFailures( logger, TestResultType.failure );
138         printedFailures |= printTestFailures( logger, TestResultType.error );
139         printedFailures |= printTestFailures( logger, TestResultType.flake );
140         if ( printedFailures )
141         {
142             logger.info( "" );
143         }
144         logger.info( globalStats.getSummary() );
145         logger.info( "" );
146     }
147 
148     public RunStatistics getGlobalRunStatistics()
149     {
150         mergeTestHistoryResult();
151         return globalStats;
152     }
153 
154     public static DefaultReporterFactory defaultNoXml()
155     {
156         return new DefaultReporterFactory( StartupReportConfiguration.defaultNoXml() );
157     }
158 
159     /**
160      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
161      * if it only has errors or failures, then count its result based on its first run
162      *
163      * @param reportEntryList the list of test run report type for a given test
164      * @param rerunFailingTestsCount configured rerun count for failing tests
165      * @return the type of test result
166      */
167     // Use default visibility for testing
168     static TestResultType getTestResultType( List<ReportEntryType> reportEntryList, int rerunFailingTestsCount  )
169     {
170         if ( reportEntryList == null || reportEntryList.isEmpty() )
171         {
172             return TestResultType.unknown;
173         }
174 
175         boolean seenSuccess = false, seenFailure = false, seenError = false;
176         for ( ReportEntryType resultType : reportEntryList )
177         {
178             if ( resultType == ReportEntryType.SUCCESS )
179             {
180                 seenSuccess = true;
181             }
182             else if ( resultType == ReportEntryType.FAILURE )
183             {
184                 seenFailure = true;
185             }
186             else if ( resultType == ReportEntryType.ERROR )
187             {
188                 seenError = true;
189             }
190         }
191 
192         if ( seenFailure || seenError )
193         {
194             if ( seenSuccess && rerunFailingTestsCount > 0 )
195             {
196                 return TestResultType.flake;
197             }
198             else
199             {
200                 if ( seenError )
201                 {
202                     return TestResultType.error;
203                 }
204                 else
205                 {
206                     return TestResultType.failure;
207                 }
208             }
209         }
210         else if ( seenSuccess )
211         {
212             return TestResultType.success;
213         }
214         else
215         {
216             return TestResultType.skipped;
217         }
218     }
219 
220     /**
221      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
222      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
223      */
224     void mergeTestHistoryResult()
225     {
226         globalStats = new RunStatistics();
227         flakyTests = new TreeMap<String, List<TestMethodStats>>();
228         failedTests = new TreeMap<String, List<TestMethodStats>>();
229         errorTests = new TreeMap<String, List<TestMethodStats>>();
230 
231         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<String, List<TestMethodStats>>();
232         // Merge all the stats for tests from listeners
233         for ( TestSetRunListener listener : listeners )
234         {
235             List<TestMethodStats> testMethodStats = listener.getTestMethodStats();
236             for ( TestMethodStats methodStats : testMethodStats )
237             {
238                 List<TestMethodStats> currentMethodStats =
239                     mergedTestHistoryResult.get( methodStats.getTestClassMethodName() );
240                 if ( currentMethodStats == null )
241                 {
242                     currentMethodStats = new ArrayList<TestMethodStats>();
243                     currentMethodStats.add( methodStats );
244                     mergedTestHistoryResult.put( methodStats.getTestClassMethodName(), currentMethodStats );
245                 }
246                 else
247                 {
248                     currentMethodStats.add( methodStats );
249                 }
250             }
251         }
252 
253         // Update globalStatistics by iterating through mergedTestHistoryResult
254         int completedCount = 0, skipped = 0;
255 
256         for ( Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet() )
257         {
258             List<TestMethodStats> testMethodStats = entry.getValue();
259             String testClassMethodName = entry.getKey();
260             completedCount++;
261 
262             List<ReportEntryType> resultTypeList = new ArrayList<ReportEntryType>();
263             for ( TestMethodStats methodStats : testMethodStats )
264             {
265                 resultTypeList.add( methodStats.getResultType() );
266             }
267 
268             TestResultType resultType = getTestResultType( resultTypeList,
269                                                            reportConfiguration.getRerunFailingTestsCount() );
270 
271             switch ( resultType )
272             {
273                 case success:
274                     // If there are multiple successful runs of the same test, count all of them
275                     int successCount = 0;
276                     for ( ReportEntryType type : resultTypeList )
277                     {
278                         if ( type == ReportEntryType.SUCCESS )
279                         {
280                             successCount++;
281                         }
282                     }
283                     completedCount += successCount - 1;
284                     break;
285                 case skipped:
286                     skipped++;
287                     break;
288                 case flake:
289                     flakyTests.put( testClassMethodName, testMethodStats );
290                     break;
291                 case failure:
292                     failedTests.put( testClassMethodName, testMethodStats );
293                     break;
294                 case error:
295                     errorTests.put( testClassMethodName, testMethodStats );
296                     break;
297                 default:
298                     throw new IllegalStateException( "Get unknown test result type" );
299             }
300         }
301 
302         globalStats.set( completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size() );
303     }
304 
305     /**
306      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
307      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
308      *
309      * @param logger the logger used to log information
310      * @param type   the type of results to be printed, could be error, failure or flake
311      * @return {@code true} if printed some lines
312      */
313     // Use default visibility for testing
314     boolean printTestFailures( DefaultDirectConsoleReporter logger, TestResultType type )
315     {
316         boolean printed = false;
317         final Map<String, List<TestMethodStats>> testStats;
318         switch ( type )
319         {
320             case failure:
321                 testStats = failedTests;
322                 break;
323             case error:
324                 testStats = errorTests;
325                 break;
326             case flake:
327                 testStats = flakyTests;
328                 break;
329             default:
330                 return printed;
331         }
332 
333         if ( !testStats.isEmpty() )
334         {
335             logger.info( type.getLogPrefix() );
336             printed = true;
337         }
338 
339         for ( Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet() )
340         {
341             printed = true;
342             List<TestMethodStats> testMethodStats = entry.getValue();
343             if ( testMethodStats.size() == 1 )
344             {
345                 // No rerun, follow the original output format
346                 logger.info( "  " + testMethodStats.get( 0 ).getStackTraceWriter().smartTrimmedStackTrace() );
347             }
348             else
349             {
350                 logger.info( entry.getKey() );
351                 for ( int i = 0; i < testMethodStats.size(); i++ )
352                 {
353                     StackTraceWriter failureStackTrace = testMethodStats.get( i ).getStackTraceWriter();
354                     if ( failureStackTrace == null )
355                     {
356                         logger.info( "  Run " + ( i + 1 ) + ": PASS" );
357                     }
358                     else
359                     {
360                         logger.info( "  Run " + ( i + 1 ) + ": " + failureStackTrace.smartTrimmedStackTrace() );
361                     }
362                 }
363                 logger.info( "" );
364             }
365         }
366         return printed;
367     }
368 
369     // Describe the result of a given test
370     static enum TestResultType
371     {
372 
373         error( "Tests in error: " ),
374         failure( "Failed tests: " ),
375         flake( "Flaked tests: " ),
376         success( "Success: " ),
377         skipped( "Skipped: " ),
378         unknown( "Unknown: " );
379 
380         private final String logPrefix;
381 
382         private TestResultType( String logPrefix )
383         {
384             this.logPrefix = logPrefix;
385         }
386 
387         public String getLogPrefix()
388         {
389             return logPrefix;
390         }
391     }
392 }