View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.plugin.surefire.report;
20  
21  import java.io.File;
22  import java.util.ArrayDeque;
23  import java.util.ArrayList;
24  import java.util.Collection;
25  import java.util.List;
26  import java.util.Queue;
27  
28  import junit.framework.TestCase;
29  import org.apache.maven.plugin.surefire.StartupReportConfiguration;
30  import org.apache.maven.plugin.surefire.extensions.SurefireConsoleOutputReporter;
31  import org.apache.maven.plugin.surefire.extensions.SurefireStatelessReporter;
32  import org.apache.maven.plugin.surefire.extensions.SurefireStatelessTestsetInfoReporter;
33  import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
34  import org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
35  import org.apache.maven.surefire.api.report.SafeThrowable;
36  import org.apache.maven.surefire.api.report.StackTraceWriter;
37  import org.apache.maven.surefire.api.report.TestOutputReportEntry;
38  import org.apache.maven.surefire.api.report.TestReportListener;
39  import org.apache.maven.surefire.api.suite.RunResult;
40  import org.apache.maven.surefire.report.RunStatistics;
41  import org.apache.maven.surefire.shared.utils.logging.MessageUtils;
42  
43  import static java.nio.charset.StandardCharsets.UTF_8;
44  import static java.util.Arrays.asList;
45  import static java.util.Collections.emptyList;
46  import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.getTestResultType;
47  import static org.mockito.Mockito.mock;
48  import static org.mockito.Mockito.when;
49  import static org.powermock.reflect.Whitebox.getInternalState;
50  import static org.powermock.reflect.Whitebox.invokeMethod;
51  
52  /**
53   *
54   */
55  public class DefaultReporterFactoryTest extends TestCase {
56      private static final String TEST_ONE = "testOne";
57  
58      private static final String TEST_TWO = "testTwo";
59  
60      private static final String TEST_THREE = "testThree";
61  
62      private static final String TEST_FOUR = "testFour";
63  
64      private static final String TEST_FIVE = "testFive";
65  
66      private static final String ASSERTION_FAIL = "assertionFail";
67  
68      private static final String ERROR = "error";
69  
70      public void testMergeTestHistoryResult() throws Exception {
71          MessageUtils.setColorEnabled(false);
72          File target = new File(System.getProperty("user.dir"), "target");
73          File reportsDirectory = new File(target, "tmp5");
74          StartupReportConfiguration reportConfig = new StartupReportConfiguration(
75                  true,
76                  true,
77                  "PLAIN",
78                  false,
79                  reportsDirectory,
80                  false,
81                  null,
82                  new File(reportsDirectory, "TESTHASH"),
83                  false,
84                  1,
85                  null,
86                  null,
87                  false,
88                  new SurefireStatelessReporter(),
89                  new SurefireConsoleOutputReporter(),
90                  new SurefireStatelessTestsetInfoReporter());
91  
92          DummyTestReporter reporter = new DummyTestReporter();
93  
94          DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
95  
96          // First run, four tests failed and one passed
97          Queue<TestMethodStats> firstRunStats = new ArrayDeque<>();
98          firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
99          firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
100         firstRunStats.add(
101                 new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
102         firstRunStats.add(
103                 new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
104         firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null));
105 
106         // Second run, two tests passed
107         Queue<TestMethodStats> secondRunStats = new ArrayDeque<>();
108         secondRunStats.add(
109                 new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
110         secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null));
111         secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
112         secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null));
113 
114         // Third run, another test passed
115         Queue<TestMethodStats> thirdRunStats = new ArrayDeque<>();
116         thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null));
117         thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
118 
119         TestSetRunListener firstRunListener = mock(TestSetRunListener.class);
120         TestSetRunListener secondRunListener = mock(TestSetRunListener.class);
121         TestSetRunListener thirdRunListener = mock(TestSetRunListener.class);
122         when(firstRunListener.getTestMethodStats()).thenReturn(firstRunStats);
123         when(secondRunListener.getTestMethodStats()).thenReturn(secondRunStats);
124         when(thirdRunListener.getTestMethodStats()).thenReturn(thirdRunStats);
125 
126         factory.addListener(firstRunListener);
127         factory.addListener(secondRunListener);
128         factory.addListener(thirdRunListener);
129 
130         invokeMethod(factory, "mergeTestHistoryResult");
131         RunStatistics mergedStatistics = factory.getGlobalRunStatistics();
132 
133         // Only TEST_THREE is a failing test, other three are flaky tests
134         assertEquals(5, mergedStatistics.getCompletedCount());
135         assertEquals(1, mergedStatistics.getErrors());
136         assertEquals(0, mergedStatistics.getFailures());
137         assertEquals(3, mergedStatistics.getFlakes());
138         assertEquals(0, mergedStatistics.getSkipped());
139 
140         // Now test the result will be printed out correctly
141         factory.printTestFailures(TestResultType.FLAKE);
142         String[] expectedFlakeOutput = {
143             "Flakes: ",
144             TEST_FOUR,
145             "  Run 1: " + ASSERTION_FAIL,
146             "  Run 2: PASS",
147             "",
148             TEST_ONE,
149             "  Run 1: " + ERROR,
150             "  Run 2: " + ASSERTION_FAIL,
151             "  Run 3: PASS",
152             "",
153             TEST_TWO,
154             "  Run 1: " + ERROR,
155             "  Run 2: PASS",
156             ""
157         };
158         assertEquals(asList(expectedFlakeOutput), reporter.getMessages());
159 
160         reporter.reset();
161         factory.printTestFailures(TestResultType.ERROR);
162         String[] expectedFailureOutput = {
163             "Errors: ", TEST_THREE, "  Run 1: " + ASSERTION_FAIL, "  Run 2: " + ERROR, "  Run 3: " + ERROR, ""
164         };
165         assertEquals(asList(expectedFailureOutput), reporter.getMessages());
166 
167         reporter.reset();
168         factory.printTestFailures(TestResultType.FAILURE);
169         assertEquals(emptyList(), reporter.getMessages());
170     }
171 
172     static final class DummyTestReporter implements ConsoleLogger {
173         private final List<String> messages = new ArrayList<>();
174 
175         @Override
176         public boolean isDebugEnabled() {
177             return true;
178         }
179 
180         @Override
181         public void debug(String message) {
182             messages.add(message);
183         }
184 
185         @Override
186         public boolean isInfoEnabled() {
187             return true;
188         }
189 
190         @Override
191         public void info(String message) {
192             messages.add(message);
193         }
194 
195         @Override
196         public boolean isWarnEnabled() {
197             return true;
198         }
199 
200         @Override
201         public void warning(String message) {
202             messages.add(message);
203         }
204 
205         @Override
206         public boolean isErrorEnabled() {
207             return true;
208         }
209 
210         @Override
211         public void error(String message) {
212             messages.add(message);
213         }
214 
215         @Override
216         public void error(String message, Throwable t) {
217             messages.add(message + " " + t.getLocalizedMessage());
218         }
219 
220         @Override
221         public void error(Throwable t) {
222             messages.add(t.getLocalizedMessage());
223         }
224 
225         List<String> getMessages() {
226             return messages;
227         }
228 
229         void reset() {
230             messages.clear();
231         }
232     }
233 
234     public void testGetTestResultType() {
235         List<ReportEntryType> emptyList = new ArrayList<>();
236         assertEquals(TestResultType.UNKNOWN, getTestResultType(emptyList, 1));
237 
238         List<ReportEntryType> successList = new ArrayList<>();
239         successList.add(ReportEntryType.SUCCESS);
240         successList.add(ReportEntryType.SUCCESS);
241         assertEquals(TestResultType.SUCCESS, getTestResultType(successList, 1));
242 
243         List<ReportEntryType> failureErrorList = new ArrayList<>();
244         failureErrorList.add(ReportEntryType.FAILURE);
245         failureErrorList.add(ReportEntryType.ERROR);
246         assertEquals(TestResultType.ERROR, getTestResultType(failureErrorList, 1));
247 
248         List<ReportEntryType> errorFailureList = new ArrayList<>();
249         errorFailureList.add(ReportEntryType.ERROR);
250         errorFailureList.add(ReportEntryType.FAILURE);
251         assertEquals(TestResultType.ERROR, getTestResultType(errorFailureList, 1));
252 
253         List<ReportEntryType> flakeList = new ArrayList<>();
254         flakeList.add(ReportEntryType.SUCCESS);
255         flakeList.add(ReportEntryType.FAILURE);
256         assertEquals(TestResultType.FLAKE, getTestResultType(flakeList, 1));
257 
258         assertEquals(TestResultType.FAILURE, getTestResultType(flakeList, 0));
259 
260         flakeList = new ArrayList<>();
261         flakeList.add(ReportEntryType.ERROR);
262         flakeList.add(ReportEntryType.SUCCESS);
263         flakeList.add(ReportEntryType.FAILURE);
264         assertEquals(TestResultType.FLAKE, getTestResultType(flakeList, 1));
265 
266         assertEquals(TestResultType.ERROR, getTestResultType(flakeList, 0));
267 
268         List<ReportEntryType> skippedList = new ArrayList<>();
269         skippedList.add(ReportEntryType.SKIPPED);
270         assertEquals(TestResultType.SKIPPED, getTestResultType(skippedList, 1));
271     }
272 
273     public void testLogger() {
274         MessageUtils.setColorEnabled(false);
275         File target = new File(System.getProperty("user.dir"), "target");
276         File reportsDirectory = new File(target, "tmp6");
277         StartupReportConfiguration reportConfig = new StartupReportConfiguration(
278                 true,
279                 true,
280                 "PLAIN",
281                 false,
282                 reportsDirectory,
283                 false,
284                 null,
285                 new File(reportsDirectory, "TESTHASH"),
286                 false,
287                 1,
288                 null,
289                 null,
290                 false,
291                 new SurefireStatelessReporter(),
292                 new SurefireConsoleOutputReporter(),
293                 new SurefireStatelessTestsetInfoReporter());
294 
295         DummyTestReporter reporter = new DummyTestReporter();
296 
297         DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
298 
299         TestReportListener<TestOutputReportEntry> runListener = factory.createTestReportListener();
300 
301         assertTrue(runListener.isDebugEnabled());
302         assertTrue(runListener.isInfoEnabled());
303         assertTrue(runListener.isWarnEnabled());
304         assertTrue(runListener.isErrorEnabled());
305 
306         runListener.debug("msg");
307         assertEquals(1, reporter.getMessages().size());
308         assertEquals("msg", reporter.getMessages().get(0));
309         reporter.reset();
310 
311         runListener.info("msg\n");
312         assertEquals(1, reporter.getMessages().size());
313         assertEquals("msg", reporter.getMessages().get(0));
314         reporter.reset();
315 
316         runListener.warning("msg\r\n");
317         assertEquals(1, reporter.getMessages().size());
318         assertEquals("msg", reporter.getMessages().get(0));
319         reporter.reset();
320 
321         runListener.error("msg");
322         assertEquals(1, reporter.getMessages().size());
323         assertEquals("msg", reporter.getMessages().get(0));
324         reporter.reset();
325 
326         runListener.error("msg\n", new Exception("e"));
327         assertEquals(1, reporter.getMessages().size());
328         assertEquals("msg e", reporter.getMessages().get(0));
329         reporter.reset();
330 
331         runListener.error(new Exception("e"));
332         assertEquals(1, reporter.getMessages().size());
333         assertEquals("e", reporter.getMessages().get(0));
334         reporter.reset();
335     }
336 
337     public void testCreateReporterWithZeroStatistics() {
338         MessageUtils.setColorEnabled(false);
339         File target = new File(System.getProperty("user.dir"), "target");
340         File reportsDirectory = new File(target, "tmp7");
341         StartupReportConfiguration reportConfig = new StartupReportConfiguration(
342                 true,
343                 true,
344                 "PLAIN",
345                 false,
346                 reportsDirectory,
347                 false,
348                 null,
349                 new File(reportsDirectory, "TESTHASH"),
350                 false,
351                 0,
352                 null,
353                 null,
354                 false,
355                 new SurefireStatelessReporter(),
356                 new SurefireConsoleOutputReporter(),
357                 new SurefireStatelessTestsetInfoReporter());
358 
359         assertTrue(reportConfig.isUseFile());
360         assertTrue(reportConfig.isPrintSummary());
361         assertEquals("PLAIN", reportConfig.getReportFormat());
362         assertFalse(reportConfig.isRedirectTestOutputToFile());
363         assertEquals(reportsDirectory, reportConfig.getReportsDirectory());
364         assertFalse(reportConfig.isTrimStackTrace());
365         assertNull(reportConfig.getReportNameSuffix());
366         assertEquals(new File(reportsDirectory, "TESTHASH"), reportConfig.getStatisticsFile());
367         assertFalse(reportConfig.isRequiresRunHistory());
368         assertEquals(0, reportConfig.getRerunFailingTestsCount());
369         assertNull(reportConfig.getXsdSchemaLocation());
370         assertEquals(UTF_8, reportConfig.getEncoding());
371         assertFalse(reportConfig.isForking());
372         assertNotNull(reportConfig.getXmlReporter());
373         assertNotNull(reportConfig.getConsoleOutputReporter());
374         assertNotNull(reportConfig.getTestsetReporter());
375         assertNull(reportConfig.getStatisticsReporter());
376 
377         DummyTestReporter reporter = new DummyTestReporter();
378 
379         DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
380         assertEquals(reportsDirectory, factory.getReportsDirectory());
381 
382         TestSetRunListener runListener = (TestSetRunListener) factory.createTestReportListener();
383         Collection listeners = getInternalState(factory, "listeners");
384         assertEquals(1, listeners.size());
385         assertTrue(listeners.contains(runListener));
386 
387         assertNotNull(runListener.getTestMethodStats());
388 
389         factory.runStarting();
390 
391         factory.close();
392 
393         RunStatistics statistics = factory.getGlobalRunStatistics();
394         assertEquals(0, statistics.getCompletedCount());
395         assertEquals(new RunResult(0, 0, 0, 0), statistics.getRunResult());
396         assertEquals(0, statistics.getFailures());
397         assertEquals(0, statistics.getErrors());
398         assertEquals(0, statistics.getSkipped());
399         assertEquals(0, statistics.getFlakes());
400         assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", statistics.getSummary());
401         assertEquals(0, statistics.getCompletedCount());
402 
403         List<String> messages = reporter.getMessages();
404         assertEquals("", messages.get(0));
405         assertEquals("-------------------------------------------------------", messages.get(1));
406         assertEquals(" T E S T S", messages.get(2));
407         assertEquals("-------------------------------------------------------", messages.get(3));
408         assertEquals("", messages.get(4));
409         assertEquals("Results:", messages.get(5));
410         assertEquals("", messages.get(6));
411         assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", messages.get(7));
412         assertEquals("", messages.get(8));
413         assertEquals(9, messages.size());
414     }
415 
416     static class DummyStackTraceWriter implements StackTraceWriter {
417 
418         private final String stackTrace;
419 
420         DummyStackTraceWriter(String stackTrace) {
421             this.stackTrace = stackTrace;
422         }
423 
424         @Override
425         public String writeTraceToString() {
426             return "";
427         }
428 
429         @Override
430         public String writeTrimmedTraceToString() {
431             return "";
432         }
433 
434         @Override
435         public String smartTrimmedStackTrace() {
436             return stackTrace;
437         }
438 
439         @Override
440         public SafeThrowable getThrowable() {
441             return null;
442         }
443     }
444 }