1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.plugin.surefire.report;
20
21 import java.io.File;
22 import java.util.ArrayDeque;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.Queue;
27
28 import junit.framework.TestCase;
29 import org.apache.maven.plugin.surefire.StartupReportConfiguration;
30 import org.apache.maven.plugin.surefire.extensions.SurefireConsoleOutputReporter;
31 import org.apache.maven.plugin.surefire.extensions.SurefireStatelessReporter;
32 import org.apache.maven.plugin.surefire.extensions.SurefireStatelessTestsetInfoReporter;
33 import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
34 import org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
35 import org.apache.maven.surefire.api.report.ReporterFactoryOptions;
36 import org.apache.maven.surefire.api.report.SafeThrowable;
37 import org.apache.maven.surefire.api.report.StackTraceWriter;
38 import org.apache.maven.surefire.api.report.TestOutputReportEntry;
39 import org.apache.maven.surefire.api.report.TestReportListener;
40 import org.apache.maven.surefire.api.suite.RunResult;
41 import org.apache.maven.surefire.report.RunStatistics;
42 import org.apache.maven.surefire.shared.utils.logging.MessageUtils;
43
44 import static java.nio.charset.StandardCharsets.UTF_8;
45 import static java.util.Arrays.asList;
46 import static java.util.Collections.emptyList;
47 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.getTestResultType;
48 import static org.mockito.Mockito.mock;
49 import static org.mockito.Mockito.when;
50 import static org.powermock.reflect.Whitebox.getInternalState;
51 import static org.powermock.reflect.Whitebox.invokeMethod;
52
53
54
55
56 public class DefaultReporterFactoryTest extends TestCase {
57 private static final String TEST_ONE = "testOne";
58
59 private static final String TEST_TWO = "testTwo";
60
61 private static final String TEST_THREE = "testThree";
62
63 private static final String TEST_FOUR = "testFour";
64
65 private static final String TEST_FIVE = "testFive";
66
67 private static final String ASSERTION_FAIL = "assertionFail";
68
69 private static final String ERROR = "error";
70
71 public void testMergeTestHistoryResult() throws Exception {
72 MessageUtils.setColorEnabled(false);
73 File target = new File(System.getProperty("user.dir"), "target");
74 File reportsDirectory = new File(target, "tmp5");
75 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
76 true,
77 true,
78 "PLAIN",
79 false,
80 reportsDirectory,
81 false,
82 null,
83 new File(reportsDirectory, "TESTHASH"),
84 false,
85 1,
86 null,
87 null,
88 false,
89 true,
90 true,
91 new SurefireStatelessReporter(),
92 new SurefireConsoleOutputReporter(),
93 new SurefireStatelessTestsetInfoReporter(),
94 new ReporterFactoryOptions());
95
96 DummyTestReporter reporter = new DummyTestReporter();
97
98 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
99
100
101 Queue<TestMethodStats> firstRunStats = new ArrayDeque<>();
102 firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
103 firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
104 firstRunStats.add(
105 new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
106 firstRunStats.add(
107 new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
108 firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null));
109
110
111 Queue<TestMethodStats> secondRunStats = new ArrayDeque<>();
112 secondRunStats.add(
113 new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
114 secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null));
115 secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
116 secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null));
117
118
119 Queue<TestMethodStats> thirdRunStats = new ArrayDeque<>();
120 thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null));
121 thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
122
123 TestSetRunListener firstRunListener = mock(TestSetRunListener.class);
124 TestSetRunListener secondRunListener = mock(TestSetRunListener.class);
125 TestSetRunListener thirdRunListener = mock(TestSetRunListener.class);
126 when(firstRunListener.getTestMethodStats()).thenReturn(firstRunStats);
127 when(secondRunListener.getTestMethodStats()).thenReturn(secondRunStats);
128 when(thirdRunListener.getTestMethodStats()).thenReturn(thirdRunStats);
129
130 factory.addListener(firstRunListener);
131 factory.addListener(secondRunListener);
132 factory.addListener(thirdRunListener);
133
134 invokeMethod(factory, "mergeTestHistoryResult");
135 RunStatistics mergedStatistics = factory.getGlobalRunStatistics();
136
137
138 assertEquals(5, mergedStatistics.getCompletedCount());
139 assertEquals(1, mergedStatistics.getErrors());
140 assertEquals(0, mergedStatistics.getFailures());
141 assertEquals(3, mergedStatistics.getFlakes());
142 assertEquals(0, mergedStatistics.getSkipped());
143
144
145 factory.printTestFailures(TestResultType.FLAKE);
146 String[] expectedFlakeOutput = {
147 "Flakes: ",
148 TEST_FOUR,
149 " Run 1: " + ASSERTION_FAIL,
150 " Run 2: PASS",
151 "",
152 TEST_ONE,
153 " Run 1: " + ERROR,
154 " Run 2: " + ASSERTION_FAIL,
155 " Run 3: PASS",
156 "",
157 TEST_TWO,
158 " Run 1: " + ERROR,
159 " Run 2: PASS",
160 ""
161 };
162 assertEquals(asList(expectedFlakeOutput), reporter.getMessages());
163
164 reporter.reset();
165 factory.printTestFailures(TestResultType.ERROR);
166 String[] expectedFailureOutput = {
167 "Errors: ", TEST_THREE, " Run 1: " + ASSERTION_FAIL, " Run 2: " + ERROR, " Run 3: " + ERROR, ""
168 };
169 assertEquals(asList(expectedFailureOutput), reporter.getMessages());
170
171 reporter.reset();
172 factory.printTestFailures(TestResultType.FAILURE);
173 assertEquals(emptyList(), reporter.getMessages());
174 }
175
176 static final class DummyTestReporter implements ConsoleLogger {
177 private final List<String> messages = new ArrayList<>();
178
179 @Override
180 public boolean isDebugEnabled() {
181 return true;
182 }
183
184 @Override
185 public void debug(String message) {
186 messages.add(message);
187 }
188
189 @Override
190 public boolean isInfoEnabled() {
191 return true;
192 }
193
194 @Override
195 public void info(String message) {
196 messages.add(message);
197 }
198
199 @Override
200 public boolean isWarnEnabled() {
201 return true;
202 }
203
204 @Override
205 public void warning(String message) {
206 messages.add(message);
207 }
208
209 @Override
210 public boolean isErrorEnabled() {
211 return true;
212 }
213
214 @Override
215 public void error(String message) {
216 messages.add(message);
217 }
218
219 @Override
220 public void error(String message, Throwable t) {
221 messages.add(message + " " + t.getLocalizedMessage());
222 }
223
224 @Override
225 public void error(Throwable t) {
226 messages.add(t.getLocalizedMessage());
227 }
228
229 List<String> getMessages() {
230 return messages;
231 }
232
233 void reset() {
234 messages.clear();
235 }
236 }
237
238 public void testGetTestResultType() {
239 List<ReportEntryType> emptyList = new ArrayList<>();
240 assertEquals(TestResultType.UNKNOWN, getTestResultType(emptyList, 1));
241
242 List<ReportEntryType> successList = new ArrayList<>();
243 successList.add(ReportEntryType.SUCCESS);
244 successList.add(ReportEntryType.SUCCESS);
245 assertEquals(TestResultType.SUCCESS, getTestResultType(successList, 1));
246
247 List<ReportEntryType> failureErrorList = new ArrayList<>();
248 failureErrorList.add(ReportEntryType.FAILURE);
249 failureErrorList.add(ReportEntryType.ERROR);
250 assertEquals(TestResultType.ERROR, getTestResultType(failureErrorList, 1));
251
252 List<ReportEntryType> errorFailureList = new ArrayList<>();
253 errorFailureList.add(ReportEntryType.ERROR);
254 errorFailureList.add(ReportEntryType.FAILURE);
255 assertEquals(TestResultType.ERROR, getTestResultType(errorFailureList, 1));
256
257 List<ReportEntryType> flakeList = new ArrayList<>();
258 flakeList.add(ReportEntryType.SUCCESS);
259 flakeList.add(ReportEntryType.FAILURE);
260 assertEquals(TestResultType.FLAKE, getTestResultType(flakeList, 1));
261
262 assertEquals(TestResultType.FAILURE, getTestResultType(flakeList, 0));
263
264 flakeList = new ArrayList<>();
265 flakeList.add(ReportEntryType.ERROR);
266 flakeList.add(ReportEntryType.SUCCESS);
267 flakeList.add(ReportEntryType.FAILURE);
268 assertEquals(TestResultType.FLAKE, getTestResultType(flakeList, 1));
269
270 assertEquals(TestResultType.ERROR, getTestResultType(flakeList, 0));
271
272 List<ReportEntryType> skippedList = new ArrayList<>();
273 skippedList.add(ReportEntryType.SKIPPED);
274 assertEquals(TestResultType.SKIPPED, getTestResultType(skippedList, 1));
275 }
276
277 public void testLogger() {
278 MessageUtils.setColorEnabled(false);
279 File target = new File(System.getProperty("user.dir"), "target");
280 File reportsDirectory = new File(target, "tmp6");
281 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
282 true,
283 true,
284 "PLAIN",
285 false,
286 reportsDirectory,
287 false,
288 null,
289 new File(reportsDirectory, "TESTHASH"),
290 false,
291 1,
292 null,
293 null,
294 false,
295 true,
296 true,
297 new SurefireStatelessReporter(),
298 new SurefireConsoleOutputReporter(),
299 new SurefireStatelessTestsetInfoReporter(),
300 new ReporterFactoryOptions());
301
302 DummyTestReporter reporter = new DummyTestReporter();
303
304 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
305
306 TestReportListener<TestOutputReportEntry> runListener = factory.createTestReportListener();
307
308 assertTrue(runListener.isDebugEnabled());
309 assertTrue(runListener.isInfoEnabled());
310 assertTrue(runListener.isWarnEnabled());
311 assertTrue(runListener.isErrorEnabled());
312
313 runListener.debug("msg");
314 assertEquals(1, reporter.getMessages().size());
315 assertEquals("msg", reporter.getMessages().get(0));
316 reporter.reset();
317
318 runListener.info("msg\n");
319 assertEquals(1, reporter.getMessages().size());
320 assertEquals("msg", reporter.getMessages().get(0));
321 reporter.reset();
322
323 runListener.warning("msg\r\n");
324 assertEquals(1, reporter.getMessages().size());
325 assertEquals("msg", reporter.getMessages().get(0));
326 reporter.reset();
327
328 runListener.error("msg");
329 assertEquals(1, reporter.getMessages().size());
330 assertEquals("msg", reporter.getMessages().get(0));
331 reporter.reset();
332
333 runListener.error("msg\n", new Exception("e"));
334 assertEquals(1, reporter.getMessages().size());
335 assertEquals("msg e", reporter.getMessages().get(0));
336 reporter.reset();
337
338 runListener.error(new Exception("e"));
339 assertEquals(1, reporter.getMessages().size());
340 assertEquals("e", reporter.getMessages().get(0));
341 reporter.reset();
342 }
343
344 public void testCreateReporterWithZeroStatistics() {
345 MessageUtils.setColorEnabled(false);
346 File target = new File(System.getProperty("user.dir"), "target");
347 File reportsDirectory = new File(target, "tmp7");
348 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
349 true,
350 true,
351 "PLAIN",
352 false,
353 reportsDirectory,
354 false,
355 null,
356 new File(reportsDirectory, "TESTHASH"),
357 false,
358 0,
359 null,
360 null,
361 false,
362 true,
363 true,
364 new SurefireStatelessReporter(),
365 new SurefireConsoleOutputReporter(),
366 new SurefireStatelessTestsetInfoReporter(),
367 new ReporterFactoryOptions());
368
369 assertTrue(reportConfig.isUseFile());
370 assertTrue(reportConfig.isPrintSummary());
371 assertEquals("PLAIN", reportConfig.getReportFormat());
372 assertFalse(reportConfig.isRedirectTestOutputToFile());
373 assertEquals(reportsDirectory, reportConfig.getReportsDirectory());
374 assertFalse(reportConfig.isTrimStackTrace());
375 assertNull(reportConfig.getReportNameSuffix());
376 assertEquals(new File(reportsDirectory, "TESTHASH"), reportConfig.getStatisticsFile());
377 assertFalse(reportConfig.isRequiresRunHistory());
378 assertEquals(0, reportConfig.getRerunFailingTestsCount());
379 assertNull(reportConfig.getXsdSchemaLocation());
380 assertEquals(UTF_8, reportConfig.getEncoding());
381 assertFalse(reportConfig.isForking());
382 assertNotNull(reportConfig.getXmlReporter());
383 assertNotNull(reportConfig.getConsoleOutputReporter());
384 assertNotNull(reportConfig.getTestsetReporter());
385 assertNull(reportConfig.getStatisticsReporter());
386
387 DummyTestReporter reporter = new DummyTestReporter();
388
389 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
390 assertEquals(reportsDirectory, factory.getReportsDirectory());
391
392 TestSetRunListener runListener = (TestSetRunListener) factory.createTestReportListener();
393 Collection listeners = getInternalState(factory, "listeners");
394 assertEquals(1, listeners.size());
395 assertTrue(listeners.contains(runListener));
396
397 assertNotNull(runListener.getTestMethodStats());
398
399 factory.runStarting();
400
401 factory.close();
402
403 RunStatistics statistics = factory.getGlobalRunStatistics();
404 assertEquals(0, statistics.getCompletedCount());
405 assertEquals(new RunResult(0, 0, 0, 0), statistics.getRunResult());
406 assertEquals(0, statistics.getFailures());
407 assertEquals(0, statistics.getErrors());
408 assertEquals(0, statistics.getSkipped());
409 assertEquals(0, statistics.getFlakes());
410 assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", statistics.getSummary());
411 assertEquals(0, statistics.getCompletedCount());
412
413 List<String> messages = reporter.getMessages();
414 assertEquals("", messages.get(0));
415 assertEquals("-------------------------------------------------------", messages.get(1));
416 assertEquals(" T E S T S", messages.get(2));
417 assertEquals("-------------------------------------------------------", messages.get(3));
418 assertEquals("", messages.get(4));
419 assertEquals("Results:", messages.get(5));
420 assertEquals("", messages.get(6));
421 assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", messages.get(7));
422 assertEquals("", messages.get(8));
423 assertEquals(9, messages.size());
424 }
425
426 static class DummyStackTraceWriter implements StackTraceWriter {
427
428 private final String stackTrace;
429
430 DummyStackTraceWriter(String stackTrace) {
431 this.stackTrace = stackTrace;
432 }
433
434 @Override
435 public String writeTraceToString() {
436 return "";
437 }
438
439 @Override
440 public String writeTrimmedTraceToString() {
441 return "";
442 }
443
444 @Override
445 public String smartTrimmedStackTrace() {
446 return stackTrace;
447 }
448
449 @Override
450 public SafeThrowable getThrowable() {
451 return null;
452 }
453 }
454 }