View Javadoc
1   package org.apache.maven.plugin.surefire.report;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
23  import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
24  import org.apache.maven.surefire.report.DefaultDirectConsoleReporter;
25  import org.apache.maven.surefire.report.ReporterFactory;
26  import org.apache.maven.surefire.report.RunListener;
27  import org.apache.maven.surefire.report.RunStatistics;
28  import org.apache.maven.surefire.report.StackTraceWriter;
29  import org.apache.maven.surefire.suite.RunResult;
30  
31  import java.util.ArrayList;
32  import java.util.Collection;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.TreeMap;
37  import java.util.concurrent.ConcurrentLinkedQueue;
38  
39  /**
40   * Provides reporting modules on the plugin side.
41   * <p/>
42   * Keeps a centralized count of test run results.
43   *
44   * @author Kristian Rosenvold
45   */
46  public class DefaultReporterFactory
47      implements ReporterFactory
48  {
49  
50      private RunStatistics globalStats = new RunStatistics();
51  
52      private final StartupReportConfiguration reportConfiguration;
53  
54      private final StatisticsReporter statisticsReporter;
55  
56      private final Collection<TestSetRunListener> listeners = new ConcurrentLinkedQueue<TestSetRunListener>();
57  
58      // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
59      private Map<String, List<TestMethodStats>> flakyTests;
60  
61      // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
62      private Map<String, List<TestMethodStats>> failedTests;
63  
64      // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
65      private Map<String, List<TestMethodStats>> errorTests;
66  
67      public DefaultReporterFactory( StartupReportConfiguration reportConfiguration )
68      {
69          this.reportConfiguration = reportConfiguration;
70          this.statisticsReporter = reportConfiguration.instantiateStatisticsReporter();
71      }
72  
73      public RunListener createReporter()
74      {
75          return createTestSetRunListener();
76      }
77  
78      public void mergeFromOtherFactories( Collection<DefaultReporterFactory> factories )
79      {
80          for ( DefaultReporterFactory factory : factories )
81          {
82              for ( TestSetRunListener listener : factory.listeners )
83              {
84                  listeners.add( listener );
85              }
86          }
87      }
88  
89      public RunListener createTestSetRunListener()
90      {
91          TestSetRunListener testSetRunListener =
92              new TestSetRunListener( reportConfiguration.instantiateConsoleReporter(),
93                                      reportConfiguration.instantiateFileReporter(),
94                                      reportConfiguration.instantiateStatelessXmlReporter(),
95                                      reportConfiguration.instantiateConsoleOutputFileReporter(), statisticsReporter,
96                                      reportConfiguration.isTrimStackTrace(),
97                                      ConsoleReporter.PLAIN.equals( reportConfiguration.getReportFormat() ),
98                                      reportConfiguration.isBriefOrPlainFormat() );
99          addListener( testSetRunListener );
100         return testSetRunListener;
101     }
102 
103     final void addListener( TestSetRunListener listener )
104     {
105         listeners.add( listener );
106     }
107 
108     public RunResult close()
109     {
110         mergeTestHistoryResult();
111         runCompleted();
112         for ( TestSetRunListener listener : listeners )
113         {
114             listener.close();
115         }
116         return globalStats.getRunResult();
117     }
118 
119     private DefaultDirectConsoleReporter createConsoleLogger()
120     {
121         return new DefaultDirectConsoleReporter( reportConfiguration.getOriginalSystemOut() );
122     }
123 
124     public void runStarting()
125     {
126         final DefaultDirectConsoleReporter consoleReporter = createConsoleLogger();
127         consoleReporter.info( "" );
128         consoleReporter.info( "-------------------------------------------------------" );
129         consoleReporter.info( " T E S T S" );
130         consoleReporter.info( "-------------------------------------------------------" );
131     }
132 
133     private void runCompleted()
134     {
135         final DefaultDirectConsoleReporter logger = createConsoleLogger();
136         if ( reportConfiguration.isPrintSummary() )
137         {
138             logger.info( "" );
139             logger.info( "Results :" );
140             logger.info( "" );
141         }
142         boolean printedFailures = printTestFailures( logger, TestResultType.failure );
143         printedFailures |= printTestFailures( logger, TestResultType.error );
144         printedFailures |= printTestFailures( logger, TestResultType.flake );
145         if ( printedFailures )
146         {
147             logger.info( "" );
148         }
149         logger.info( globalStats.getSummary() );
150         logger.info( "" );
151     }
152 
153     public RunStatistics getGlobalRunStatistics()
154     {
155         mergeTestHistoryResult();
156         return globalStats;
157     }
158 
159     public static DefaultReporterFactory defaultNoXml()
160     {
161         return new DefaultReporterFactory( StartupReportConfiguration.defaultNoXml() );
162     }
163 
164     /**
165      * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
166      * if it only has errors or failures, then count its result based on its first run
167      *
168      * @param reportEntryList the list of test run report type for a given test
169      * @param rerunFailingTestsCount configured rerun count for failing tests
170      * @return the type of test result
171      */
172     // Use default visibility for testing
173     static TestResultType getTestResultType( List<ReportEntryType> reportEntryList, int rerunFailingTestsCount  )
174     {
175         if ( reportEntryList == null || reportEntryList.isEmpty() )
176         {
177             return TestResultType.unknown;
178         }
179 
180         boolean seenSuccess = false, seenFailure = false, seenError = false;
181         for ( ReportEntryType resultType : reportEntryList )
182         {
183             if ( resultType == ReportEntryType.SUCCESS )
184             {
185                 seenSuccess = true;
186             }
187             else if ( resultType == ReportEntryType.FAILURE )
188             {
189                 seenFailure = true;
190             }
191             else if ( resultType == ReportEntryType.ERROR )
192             {
193                 seenError = true;
194             }
195         }
196 
197         if ( seenFailure || seenError )
198         {
199             if ( seenSuccess && rerunFailingTestsCount > 0 )
200             {
201                 return TestResultType.flake;
202             }
203             else
204             {
205                 if ( seenError )
206                 {
207                     return TestResultType.error;
208                 }
209                 else
210                 {
211                     return TestResultType.failure;
212                 }
213             }
214         }
215         else if ( seenSuccess )
216         {
217             return TestResultType.success;
218         }
219         else
220         {
221             return TestResultType.skipped;
222         }
223     }
224 
225     /**
226      * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and
227      * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge.
228      */
229     void mergeTestHistoryResult()
230     {
231         globalStats = new RunStatistics();
232         flakyTests = new TreeMap<String, List<TestMethodStats>>();
233         failedTests = new TreeMap<String, List<TestMethodStats>>();
234         errorTests = new TreeMap<String, List<TestMethodStats>>();
235 
236         Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<String, List<TestMethodStats>>();
237         // Merge all the stats for tests from listeners
238         for ( TestSetRunListener listener : listeners )
239         {
240             List<TestMethodStats> testMethodStats = listener.getTestMethodStats();
241             for ( TestMethodStats methodStats : testMethodStats )
242             {
243                 List<TestMethodStats> currentMethodStats =
244                     mergedTestHistoryResult.get( methodStats.getTestClassMethodName() );
245                 if ( currentMethodStats == null )
246                 {
247                     currentMethodStats = new ArrayList<TestMethodStats>();
248                     currentMethodStats.add( methodStats );
249                     mergedTestHistoryResult.put( methodStats.getTestClassMethodName(), currentMethodStats );
250                 }
251                 else
252                 {
253                     currentMethodStats.add( methodStats );
254                 }
255             }
256         }
257 
258         // Update globalStatistics by iterating through mergedTestHistoryResult
259         int completedCount = 0, skipped = 0;
260 
261         for ( Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet() )
262         {
263             List<TestMethodStats> testMethodStats = entry.getValue();
264             String testClassMethodName = entry.getKey();
265             completedCount++;
266 
267             List<ReportEntryType> resultTypeList = new ArrayList<ReportEntryType>();
268             for ( TestMethodStats methodStats : testMethodStats )
269             {
270                 resultTypeList.add( methodStats.getResultType() );
271             }
272 
273             TestResultType resultType = getTestResultType( resultTypeList,
274                                                            reportConfiguration.getRerunFailingTestsCount() );
275 
276             switch ( resultType )
277             {
278                 case success:
279                     // If there are multiple successful runs of the same test, count all of them
280                     int successCount = 0;
281                     for ( ReportEntryType type : resultTypeList )
282                     {
283                         if ( type == ReportEntryType.SUCCESS )
284                         {
285                             successCount++;
286                         }
287                     }
288                     completedCount += successCount - 1;
289                     break;
290                 case skipped:
291                     skipped++;
292                     break;
293                 case flake:
294                     flakyTests.put( testClassMethodName, testMethodStats );
295                     break;
296                 case failure:
297                     failedTests.put( testClassMethodName, testMethodStats );
298                     break;
299                 case error:
300                     errorTests.put( testClassMethodName, testMethodStats );
301                     break;
302                 default:
303                     throw new IllegalStateException( "Get unknown test result type" );
304             }
305         }
306 
307         globalStats.set( completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size() );
308     }
309 
310     /**
311      * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
312      * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
313      *
314      * @param logger the logger used to log information
315      * @param type   the type of results to be printed, could be error, failure or flake
316      * @return {@code true} if printed some lines
317      */
318     // Use default visibility for testing
319     boolean printTestFailures( DefaultDirectConsoleReporter logger, TestResultType type )
320     {
321         boolean printed = false;
322         final Map<String, List<TestMethodStats>> testStats;
323         switch ( type )
324         {
325             case failure:
326                 testStats = failedTests;
327                 break;
328             case error:
329                 testStats = errorTests;
330                 break;
331             case flake:
332                 testStats = flakyTests;
333                 break;
334             default:
335                 return printed;
336         }
337 
338         if ( !testStats.isEmpty() )
339         {
340             logger.info( type.getLogPrefix() );
341             printed = true;
342         }
343 
344         for ( Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet() )
345         {
346             printed = true;
347             List<TestMethodStats> testMethodStats = entry.getValue();
348             if ( testMethodStats.size() == 1 )
349             {
350                 // No rerun, follow the original output format
351                 logger.info( "  " + testMethodStats.get( 0 ).getStackTraceWriter().smartTrimmedStackTrace() );
352             }
353             else
354             {
355                 logger.info( entry.getKey() );
356                 for ( int i = 0; i < testMethodStats.size(); i++ )
357                 {
358                     StackTraceWriter failureStackTrace = testMethodStats.get( i ).getStackTraceWriter();
359                     if ( failureStackTrace == null )
360                     {
361                         logger.info( "  Run " + ( i + 1 ) + ": PASS" );
362                     }
363                     else
364                     {
365                         logger.info( "  Run " + ( i + 1 ) + ": " + failureStackTrace.smartTrimmedStackTrace() );
366                     }
367                 }
368                 logger.info( "" );
369             }
370         }
371         return printed;
372     }
373 
374     // Describe the result of a given test
375     static enum TestResultType
376     {
377 
378         error( "Tests in error: " ),
379         failure( "Failed tests: " ),
380         flake( "Flaked tests: " ),
381         success( "Success: " ),
382         skipped( "Skipped: " ),
383         unknown( "Unknown: " );
384 
385         private final String logPrefix;
386 
387         private TestResultType( String logPrefix )
388         {
389             this.logPrefix = logPrefix;
390         }
391 
392         public String getLogPrefix()
393         {
394             return logPrefix;
395         }
396     }
397 }