View Javadoc
1   package org.apache.maven.plugin.surefire.report;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
23  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
24  import org.apache.maven.surefire.report.DefaultDirectConsoleReporter;
25  import org.apache.maven.surefire.report.ReporterFactory;
26  import org.apache.maven.surefire.report.RunListener;
27  import org.apache.maven.surefire.report.RunStatistics;
28  import org.apache.maven.surefire.report.StackTraceWriter;
29  import org.apache.maven.surefire.suite.RunResult;
30  
31  import java.util.ArrayList;
32  import java.util.Collections;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.TreeMap;
37  
38  /**
39   * Provides reporting modules on the plugin side.
40   * <p/>
41   * Keeps a centralized count of test run results.
42   *
43   * @author Kristian Rosenvold
44   */
45  public class DefaultReporterFactory
46      implements ReporterFactory
47  {
48  
49      private RunStatistics globalStats = new RunStatistics();
50  
51      private final StartupReportConfiguration reportConfiguration;
52  
53      private final StatisticsReporter statisticsReporter;
54  
55      private final List<TestSetRunListener> listeners =
56          Collections.synchronizedList( new ArrayList<TestSetRunListener>() );
57  
58      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
59      private Map<String, List<TestMethodStats>> flakyTests = null;
60  
61      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
62      private Map<String, List<TestMethodStats>> failedTests = null;
63  
64      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
65      private Map<String, List<TestMethodStats>> errorTests = null;
66  
67      public DefaultReporterFactory( StartupReportConfiguration reportConfiguration )
68      {
69          this.reportConfiguration = reportConfiguration;
70          this.statisticsReporter = reportConfiguration.instantiateStatisticsReporter();
71      }
72  
73      public RunListener createReporter()
74      {
75          return createTestSetRunListener();
76      }
77  
78      public void mergeFromOtherFactories( List<DefaultReporterFactory> factories )
79      {
80          for ( DefaultReporterFactory factory : factories )
81          {
82              for ( TestSetRunListener listener : factory.listeners )
83              {
84                  listeners.add( listener );
85              }
86          }
87      }
88  
89      public RunListener createTestSetRunListener()
90      {
91          TestSetRunListener testSetRunListener =
92              new TestSetRunListener( reportConfiguration.instantiateConsoleReporter(),
93                                      reportConfiguration.instantiateFileReporter(),
94                                      reportConfiguration.instantiateStatelessXmlReporter(),
95                                      reportConfiguration.instantiateConsoleOutputFileReporter(), statisticsReporter,
96                                      reportConfiguration.isTrimStackTrace(),
97                                      ConsoleReporter.PLAIN.equals( reportConfiguration.getReportFormat() ),
98                                      reportConfiguration.isBriefOrPlainFormat() );
99          listeners.add( testSetRunListener );
100         return testSetRunListener;
101     }
102 
103     public void addListener( TestSetRunListener listener )
104     {
105         listeners.add( listener );
106     }
107 
108     public RunResult close()
109     {
110         mergeTestHistoryResult();
111         runCompleted();
112         for ( TestSetRunListener listener : listeners )
113         {
114             listener.close();
115         }
116         return globalStats.getRunResult();
117     }
118 
119     private DefaultDirectConsoleReporter createConsoleLogger()
120     {
121         return new DefaultDirectConsoleReporter( reportConfiguration.getOriginalSystemOut() );
122     }
123 
124     public void runStarting()
125     {
126         final DefaultDirectConsoleReporter consoleReporter = createConsoleLogger();
127         consoleReporter.info( "" );
128         consoleReporter.info( "-------------------------------------------------------" );
129         consoleReporter.info( " T E S T S" );
130         consoleReporter.info( "-------------------------------------------------------" );
131     }
132 
133     private void runCompleted()
134     {
135         final DefaultDirectConsoleReporter logger = createConsoleLogger();
136         if ( reportConfiguration.isPrintSummary() )
137         {
138             logger.info( "" );
139             logger.info( "Results :" );
140             logger.info( "" );
141         }
142         printTestFailures( logger, TestResultType.failure );
143         printTestFailures( logger, TestResultType.error );
144         printTestFailures( logger, TestResultType.flake );
145         logger.info( globalStats.getSummary() );
146         logger.info( "" );
147     }
148 
149     public RunStatistics getGlobalRunStatistics()
150     {
151         mergeTestHistoryResult();
152         return globalStats;
153     }
154 
155     public static DefaultReporterFactory defaultNoXml()
156     {
157         return new DefaultReporterFactory( StartupReportConfiguration.defaultNoXml() );
158     }
159 
160     /**
161      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
162      * if it only has errors or failures, then count its result based on its first run
163      *
164      * @param reportEntryList the list of test run report type for a given test
165      * @param rerunFailingTestsCount configured rerun count for failing tests
166      * @return the type of test result
167      */
168     // Use default visibility for testing
169     static TestResultType getTestResultType( List<ReportEntryType> reportEntryList, int rerunFailingTestsCount  )
170     {
171         if ( reportEntryList == null || reportEntryList.size() == 0 )
172         {
173             return TestResultType.unknown;
174         }
175 
176         boolean seenSuccess = false, seenFailure = false, seenError = false;
177         for ( ReportEntryType resultType : reportEntryList )
178         {
179             if ( resultType == ReportEntryType.SUCCESS )
180             {
181                 seenSuccess = true;
182             }
183             else if ( resultType == ReportEntryType.FAILURE )
184             {
185                 seenFailure = true;
186             }
187             else if ( resultType == ReportEntryType.ERROR )
188             {
189                 seenError = true;
190             }
191         }
192 
193         if ( seenFailure || seenError )
194         {
195             if ( seenSuccess && rerunFailingTestsCount > 0 )
196             {
197                 return TestResultType.flake;
198             }
199             else
200             {
201                 if ( seenError )
202                 {
203                     return TestResultType.error;
204                 }
205                 else if ( seenFailure )
206                 {
207                     return TestResultType.failure;
208                 }
209                 else
210                 {
211                     return TestResultType.skipped;
212                 }
213             }
214         }
215         else if ( seenSuccess )
216         {
217             return TestResultType.success;
218         }
219         else
220         {
221             return TestResultType.skipped;
222         }
223     }
224 
225     /**
226      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
227      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
228      */
229     void mergeTestHistoryResult()
230     {
231         globalStats = new RunStatistics();
232         flakyTests = new TreeMap<String, List<TestMethodStats>>();
233         failedTests = new TreeMap<String, List<TestMethodStats>>();
234         errorTests = new TreeMap<String, List<TestMethodStats>>();
235 
236         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<String, List<TestMethodStats>>();
237         // Merge all the stats for tests from listeners
238         for ( TestSetRunListener listener : listeners )
239         {
240             List<TestMethodStats> testMethodStats = listener.getTestMethodStats();
241             for ( TestMethodStats methodStats : testMethodStats )
242             {
243                 List<TestMethodStats> currentMethodStats =
244                     mergedTestHistoryResult.get( methodStats.getTestClassMethodName() );
245                 if ( currentMethodStats == null )
246                 {
247                     currentMethodStats = new ArrayList<TestMethodStats>();
248                     currentMethodStats.add( methodStats );
249                     mergedTestHistoryResult.put( methodStats.getTestClassMethodName(), currentMethodStats );
250                 }
251                 else
252                 {
253                     currentMethodStats.add( methodStats );
254                 }
255             }
256         }
257 
258         // Update globalStatistics by iterating through mergedTestHistoryResult
259         int completedCount = 0, skipped = 0;
260 
261         for ( Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet() )
262         {
263             List<TestMethodStats> testMethodStats = entry.getValue();
264             String testClassMethodName = entry.getKey();
265             completedCount++;
266 
267             List<ReportEntryType> resultTypeList = new ArrayList<ReportEntryType>();
268             for ( TestMethodStats methodStats : testMethodStats )
269             {
270                 resultTypeList.add( methodStats.getResultType() );
271             }
272 
273             TestResultType resultType = getTestResultType( resultTypeList,
274                                                            reportConfiguration.getRerunFailingTestsCount() );
275 
276             switch ( resultType )
277             {
278                 case success:
279                     // If there are multiple successful runs of the same test, count all of them
280                     int successCount = 0;
281                     for ( ReportEntryType type : resultTypeList )
282                     {
283                         if ( type == ReportEntryType.SUCCESS )
284                         {
285                             successCount++;
286                         }
287                     }
288                     completedCount += successCount - 1;
289                     break;
290                 case skipped:
291                     skipped++;
292                     break;
293                 case flake:
294                     flakyTests.put( testClassMethodName, testMethodStats );
295                     break;
296                 case failure:
297                     failedTests.put( testClassMethodName, testMethodStats );
298                     break;
299                 case error:
300                     errorTests.put( testClassMethodName, testMethodStats );
301                     break;
302                 default:
303                     throw new IllegalStateException( "Get unknown test result type" );
304             }
305         }
306 
307         globalStats.set( completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size() );
308     }
309 
310     /**
311      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
312      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
313      *
314      * @param logger the logger used to log information
315      * @param type   the type of results to be printed, could be error, failure or flake
316      */
317     // Use default visibility for testing
318     void printTestFailures( DefaultDirectConsoleReporter logger, TestResultType type )
319     {
320         Map<String, List<TestMethodStats>> testStats;
321         if ( type == TestResultType.failure )
322         {
323             testStats = failedTests;
324         }
325         else if ( type == TestResultType.error )
326         {
327             testStats = errorTests;
328         }
329         else if ( type == TestResultType.flake )
330         {
331             testStats = flakyTests;
332         }
333         else
334         {
335             logger.info( "" );
336             return;
337         }
338 
339         if ( testStats.size() > 0 )
340         {
341             logger.info( type.getLogPrefix() );
342         }
343 
344         for ( Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet() )
345         {
346             List<TestMethodStats> testMethodStats = entry.getValue();
347             if ( testMethodStats.size() == 1 )
348             {
349                 // No rerun, follow the original output format
350                 logger.info( "  " + testMethodStats.get( 0 ).getStackTraceWriter().smartTrimmedStackTrace() );
351                 continue;
352             }
353 
354             logger.info( entry.getKey() );
355 
356             for ( int i = 0; i < testMethodStats.size(); i++ )
357             {
358                 StackTraceWriter failureStackTrace = testMethodStats.get( i ).getStackTraceWriter();
359                 if ( failureStackTrace == null )
360                 {
361                     logger.info( "  Run " + ( i + 1 ) + ": PASS" );
362                 }
363                 else
364                 {
365                     logger.info( "  Run " + ( i + 1 ) + ": " + failureStackTrace.smartTrimmedStackTrace() );
366                 }
367             }
368             logger.info( "" );
369         }
370         logger.info( "" );
371     }
372 
373     // Describe the result of a given test
374     static enum TestResultType
375     {
376 
377         error( "Tests in error: " ),
378         failure( "Failed tests: " ),
379         flake( "Flaked tests: " ),
380         success( "Success: " ),
381         skipped( "Skipped: " ),
382         unknown( "Unknown: " );
383 
384         private final String logPrefix;
385 
386         private TestResultType( String logPrefix )
387         {
388             this.logPrefix = logPrefix;
389         }
390 
391         public String getLogPrefix()
392         {
393             return logPrefix;
394         }
395     }
396 }