View Javadoc
1   package org.apache.maven.plugin.surefire.report;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
23  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
24  import org.apache.maven.surefire.report.DefaultDirectConsoleReporter;
25  import org.apache.maven.surefire.report.ReporterFactory;
26  import org.apache.maven.surefire.report.RunListener;
27  import org.apache.maven.surefire.report.RunStatistics;
28  import org.apache.maven.surefire.report.StackTraceWriter;
29  import org.apache.maven.surefire.suite.RunResult;
30  
31  import java.util.ArrayList;
32  import java.util.Collection;
33  import java.util.Collections;
34  import java.util.HashMap;
35  import java.util.List;
36  import java.util.Map;
37  import java.util.TreeMap;
38  
39  /**
40   * Provides reporting modules on the plugin side.
41   * <p/>
42   * Keeps a centralized count of test run results.
43   *
44   * @author Kristian Rosenvold
45   */
46  public class DefaultReporterFactory
47      implements ReporterFactory
48  {
49  
50      private RunStatistics globalStats = new RunStatistics();
51  
52      private final StartupReportConfiguration reportConfiguration;
53  
54      private final StatisticsReporter statisticsReporter;
55  
56      private final List<TestSetRunListener> listeners =
57          Collections.synchronizedList( new ArrayList<TestSetRunListener>() );
58  
59      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
60      private Map<String, List<TestMethodStats>> flakyTests;
61  
62      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
63      private Map<String, List<TestMethodStats>> failedTests;
64  
65      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
66      private Map<String, List<TestMethodStats>> errorTests;
67  
68      public DefaultReporterFactory( StartupReportConfiguration reportConfiguration )
69      {
70          this.reportConfiguration = reportConfiguration;
71          this.statisticsReporter = reportConfiguration.instantiateStatisticsReporter();
72      }
73  
74      public RunListener createReporter()
75      {
76          return createTestSetRunListener();
77      }
78  
79      public void mergeFromOtherFactories( Collection<DefaultReporterFactory> factories )
80      {
81          for ( DefaultReporterFactory factory : factories )
82          {
83              for ( TestSetRunListener listener : factory.listeners )
84              {
85                  listeners.add( listener );
86              }
87          }
88      }
89  
90      public RunListener createTestSetRunListener()
91      {
92          TestSetRunListener testSetRunListener =
93              new TestSetRunListener( reportConfiguration.instantiateConsoleReporter(),
94                                      reportConfiguration.instantiateFileReporter(),
95                                      reportConfiguration.instantiateStatelessXmlReporter(),
96                                      reportConfiguration.instantiateConsoleOutputFileReporter(), statisticsReporter,
97                                      reportConfiguration.isTrimStackTrace(),
98                                      ConsoleReporter.PLAIN.equals( reportConfiguration.getReportFormat() ),
99                                      reportConfiguration.isBriefOrPlainFormat() );
100         listeners.add( testSetRunListener );
101         return testSetRunListener;
102     }
103 
104     public void addListener( TestSetRunListener listener )
105     {
106         listeners.add( listener );
107     }
108 
109     public RunResult close()
110     {
111         mergeTestHistoryResult();
112         runCompleted();
113         for ( TestSetRunListener listener : listeners )
114         {
115             listener.close();
116         }
117         return globalStats.getRunResult();
118     }
119 
120     private DefaultDirectConsoleReporter createConsoleLogger()
121     {
122         return new DefaultDirectConsoleReporter( reportConfiguration.getOriginalSystemOut() );
123     }
124 
125     public void runStarting()
126     {
127         final DefaultDirectConsoleReporter consoleReporter = createConsoleLogger();
128         consoleReporter.info( "" );
129         consoleReporter.info( "-------------------------------------------------------" );
130         consoleReporter.info( " T E S T S" );
131         consoleReporter.info( "-------------------------------------------------------" );
132     }
133 
134     private void runCompleted()
135     {
136         final DefaultDirectConsoleReporter logger = createConsoleLogger();
137         if ( reportConfiguration.isPrintSummary() )
138         {
139             logger.info( "" );
140             logger.info( "Results :" );
141             logger.info( "" );
142         }
143         boolean printedFailures = printTestFailures( logger, TestResultType.failure );
144         printedFailures |= printTestFailures( logger, TestResultType.error );
145         printedFailures |= printTestFailures( logger, TestResultType.flake );
146         if ( printedFailures )
147         {
148             logger.info( "" );
149         }
150         logger.info( globalStats.getSummary() );
151         logger.info( "" );
152     }
153 
154     public RunStatistics getGlobalRunStatistics()
155     {
156         mergeTestHistoryResult();
157         return globalStats;
158     }
159 
160     public static DefaultReporterFactory defaultNoXml()
161     {
162         return new DefaultReporterFactory( StartupReportConfiguration.defaultNoXml() );
163     }
164 
165     /**
166      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
167      * if it only has errors or failures, then count its result based on its first run
168      *
169      * @param reportEntryList the list of test run report type for a given test
170      * @param rerunFailingTestsCount configured rerun count for failing tests
171      * @return the type of test result
172      */
173     // Use default visibility for testing
174     static TestResultType getTestResultType( List<ReportEntryType> reportEntryList, int rerunFailingTestsCount  )
175     {
176         if ( reportEntryList == null || reportEntryList.isEmpty() )
177         {
178             return TestResultType.unknown;
179         }
180 
181         boolean seenSuccess = false, seenFailure = false, seenError = false;
182         for ( ReportEntryType resultType : reportEntryList )
183         {
184             if ( resultType == ReportEntryType.SUCCESS )
185             {
186                 seenSuccess = true;
187             }
188             else if ( resultType == ReportEntryType.FAILURE )
189             {
190                 seenFailure = true;
191             }
192             else if ( resultType == ReportEntryType.ERROR )
193             {
194                 seenError = true;
195             }
196         }
197 
198         if ( seenFailure || seenError )
199         {
200             if ( seenSuccess && rerunFailingTestsCount > 0 )
201             {
202                 return TestResultType.flake;
203             }
204             else
205             {
206                 if ( seenError )
207                 {
208                     return TestResultType.error;
209                 }
210                 else
211                 {
212                     return TestResultType.failure;
213                 }
214             }
215         }
216         else if ( seenSuccess )
217         {
218             return TestResultType.success;
219         }
220         else
221         {
222             return TestResultType.skipped;
223         }
224     }
225 
226     /**
227      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
228      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
229      */
230     void mergeTestHistoryResult()
231     {
232         globalStats = new RunStatistics();
233         flakyTests = new TreeMap<String, List<TestMethodStats>>();
234         failedTests = new TreeMap<String, List<TestMethodStats>>();
235         errorTests = new TreeMap<String, List<TestMethodStats>>();
236 
237         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<String, List<TestMethodStats>>();
238         // Merge all the stats for tests from listeners
239         for ( TestSetRunListener listener : listeners )
240         {
241             List<TestMethodStats> testMethodStats = listener.getTestMethodStats();
242             for ( TestMethodStats methodStats : testMethodStats )
243             {
244                 List<TestMethodStats> currentMethodStats =
245                     mergedTestHistoryResult.get( methodStats.getTestClassMethodName() );
246                 if ( currentMethodStats == null )
247                 {
248                     currentMethodStats = new ArrayList<TestMethodStats>();
249                     currentMethodStats.add( methodStats );
250                     mergedTestHistoryResult.put( methodStats.getTestClassMethodName(), currentMethodStats );
251                 }
252                 else
253                 {
254                     currentMethodStats.add( methodStats );
255                 }
256             }
257         }
258 
259         // Update globalStatistics by iterating through mergedTestHistoryResult
260         int completedCount = 0, skipped = 0;
261 
262         for ( Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet() )
263         {
264             List<TestMethodStats> testMethodStats = entry.getValue();
265             String testClassMethodName = entry.getKey();
266             completedCount++;
267 
268             List<ReportEntryType> resultTypeList = new ArrayList<ReportEntryType>();
269             for ( TestMethodStats methodStats : testMethodStats )
270             {
271                 resultTypeList.add( methodStats.getResultType() );
272             }
273 
274             TestResultType resultType = getTestResultType( resultTypeList,
275                                                            reportConfiguration.getRerunFailingTestsCount() );
276 
277             switch ( resultType )
278             {
279                 case success:
280                     // If there are multiple successful runs of the same test, count all of them
281                     int successCount = 0;
282                     for ( ReportEntryType type : resultTypeList )
283                     {
284                         if ( type == ReportEntryType.SUCCESS )
285                         {
286                             successCount++;
287                         }
288                     }
289                     completedCount += successCount - 1;
290                     break;
291                 case skipped:
292                     skipped++;
293                     break;
294                 case flake:
295                     flakyTests.put( testClassMethodName, testMethodStats );
296                     break;
297                 case failure:
298                     failedTests.put( testClassMethodName, testMethodStats );
299                     break;
300                 case error:
301                     errorTests.put( testClassMethodName, testMethodStats );
302                     break;
303                 default:
304                     throw new IllegalStateException( "Get unknown test result type" );
305             }
306         }
307 
308         globalStats.set( completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size() );
309     }
310 
311     /**
312      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
313      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
314      *
315      * @param logger the logger used to log information
316      * @param type   the type of results to be printed, could be error, failure or flake
317      * @return {@code true} if printed some lines
318      */
319     // Use default visibility for testing
320     boolean printTestFailures( DefaultDirectConsoleReporter logger, TestResultType type )
321     {
322         boolean printed = false;
323         final Map<String, List<TestMethodStats>> testStats;
324         switch ( type )
325         {
326             case failure:
327                 testStats = failedTests;
328                 break;
329             case error:
330                 testStats = errorTests;
331                 break;
332             case flake:
333                 testStats = flakyTests;
334                 break;
335             default:
336                 return printed;
337         }
338 
339         if ( !testStats.isEmpty() )
340         {
341             logger.info( type.getLogPrefix() );
342             printed = true;
343         }
344 
345         for ( Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet() )
346         {
347             printed = true;
348             List<TestMethodStats> testMethodStats = entry.getValue();
349             if ( testMethodStats.size() == 1 )
350             {
351                 // No rerun, follow the original output format
352                 logger.info( "  " + testMethodStats.get( 0 ).getStackTraceWriter().smartTrimmedStackTrace() );
353             }
354             else
355             {
356                 logger.info( entry.getKey() );
357                 for ( int i = 0; i < testMethodStats.size(); i++ )
358                 {
359                     StackTraceWriter failureStackTrace = testMethodStats.get( i ).getStackTraceWriter();
360                     if ( failureStackTrace == null )
361                     {
362                         logger.info( "  Run " + ( i + 1 ) + ": PASS" );
363                     }
364                     else
365                     {
366                         logger.info( "  Run " + ( i + 1 ) + ": " + failureStackTrace.smartTrimmedStackTrace() );
367                     }
368                 }
369                 logger.info( "" );
370             }
371         }
372         return printed;
373     }
374 
375     // Describe the result of a given test
376     static enum TestResultType
377     {
378 
379         error( "Tests in error: " ),
380         failure( "Failed tests: " ),
381         flake( "Flaked tests: " ),
382         success( "Success: " ),
383         skipped( "Skipped: " ),
384         unknown( "Unknown: " );
385 
386         private final String logPrefix;
387 
388         private TestResultType( String logPrefix )
389         {
390             this.logPrefix = logPrefix;
391         }
392 
393         public String getLogPrefix()
394         {
395             return logPrefix;
396         }
397     }
398 }