1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.plugin.surefire.report;
20
21 import java.io.File;
22 import java.util.ArrayDeque;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.Queue;
27
28 import junit.framework.TestCase;
29 import org.apache.maven.plugin.surefire.StartupReportConfiguration;
30 import org.apache.maven.plugin.surefire.extensions.SurefireConsoleOutputReporter;
31 import org.apache.maven.plugin.surefire.extensions.SurefireStatelessReporter;
32 import org.apache.maven.plugin.surefire.extensions.SurefireStatelessTestsetInfoReporter;
33 import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
34 import org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
35 import org.apache.maven.surefire.api.report.SafeThrowable;
36 import org.apache.maven.surefire.api.report.StackTraceWriter;
37 import org.apache.maven.surefire.api.report.TestOutputReportEntry;
38 import org.apache.maven.surefire.api.report.TestReportListener;
39 import org.apache.maven.surefire.api.suite.RunResult;
40 import org.apache.maven.surefire.report.RunStatistics;
41 import org.apache.maven.surefire.shared.utils.logging.MessageUtils;
42
43 import static java.nio.charset.StandardCharsets.UTF_8;
44 import static java.util.Arrays.asList;
45 import static java.util.Collections.emptyList;
46 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.getTestResultType;
47 import static org.mockito.Mockito.mock;
48 import static org.mockito.Mockito.when;
49 import static org.powermock.reflect.Whitebox.getInternalState;
50 import static org.powermock.reflect.Whitebox.invokeMethod;
51
52
53
54
55 public class DefaultReporterFactoryTest extends TestCase {
56 private static final String TEST_ONE = "testOne";
57
58 private static final String TEST_TWO = "testTwo";
59
60 private static final String TEST_THREE = "testThree";
61
62 private static final String TEST_FOUR = "testFour";
63
64 private static final String TEST_FIVE = "testFive";
65
66 private static final String ASSERTION_FAIL = "assertionFail";
67
68 private static final String ERROR = "error";
69
70 public void testMergeTestHistoryResult() throws Exception {
71 MessageUtils.setColorEnabled(false);
72 File target = new File(System.getProperty("user.dir"), "target");
73 File reportsDirectory = new File(target, "tmp5");
74 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
75 true,
76 true,
77 "PLAIN",
78 false,
79 reportsDirectory,
80 false,
81 null,
82 new File(reportsDirectory, "TESTHASH"),
83 false,
84 1,
85 null,
86 null,
87 false,
88 new SurefireStatelessReporter(),
89 new SurefireConsoleOutputReporter(),
90 new SurefireStatelessTestsetInfoReporter());
91
92 DummyTestReporter reporter = new DummyTestReporter();
93
94 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
95
96
97 Queue<TestMethodStats> firstRunStats = new ArrayDeque<>();
98 firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
99 firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
100 firstRunStats.add(
101 new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
102 firstRunStats.add(
103 new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
104 firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null));
105
106
107 Queue<TestMethodStats> secondRunStats = new ArrayDeque<>();
108 secondRunStats.add(
109 new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
110 secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null));
111 secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
112 secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null));
113
114
115 Queue<TestMethodStats> thirdRunStats = new ArrayDeque<>();
116 thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null));
117 thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
118
119 TestSetRunListener firstRunListener = mock(TestSetRunListener.class);
120 TestSetRunListener secondRunListener = mock(TestSetRunListener.class);
121 TestSetRunListener thirdRunListener = mock(TestSetRunListener.class);
122 when(firstRunListener.getTestMethodStats()).thenReturn(firstRunStats);
123 when(secondRunListener.getTestMethodStats()).thenReturn(secondRunStats);
124 when(thirdRunListener.getTestMethodStats()).thenReturn(thirdRunStats);
125
126 factory.addListener(firstRunListener);
127 factory.addListener(secondRunListener);
128 factory.addListener(thirdRunListener);
129
130 invokeMethod(factory, "mergeTestHistoryResult");
131 RunStatistics mergedStatistics = factory.getGlobalRunStatistics();
132
133
134 assertEquals(5, mergedStatistics.getCompletedCount());
135 assertEquals(1, mergedStatistics.getErrors());
136 assertEquals(0, mergedStatistics.getFailures());
137 assertEquals(3, mergedStatistics.getFlakes());
138 assertEquals(0, mergedStatistics.getSkipped());
139
140
141 factory.printTestFailures(TestResultType.FLAKE);
142 String[] expectedFlakeOutput = {
143 "Flakes: ",
144 TEST_FOUR,
145 " Run 1: " + ASSERTION_FAIL,
146 " Run 2: PASS",
147 "",
148 TEST_ONE,
149 " Run 1: " + ERROR,
150 " Run 2: " + ASSERTION_FAIL,
151 " Run 3: PASS",
152 "",
153 TEST_TWO,
154 " Run 1: " + ERROR,
155 " Run 2: PASS",
156 ""
157 };
158 assertEquals(asList(expectedFlakeOutput), reporter.getMessages());
159
160 reporter.reset();
161 factory.printTestFailures(TestResultType.ERROR);
162 String[] expectedFailureOutput = {
163 "Errors: ", TEST_THREE, " Run 1: " + ASSERTION_FAIL, " Run 2: " + ERROR, " Run 3: " + ERROR, ""
164 };
165 assertEquals(asList(expectedFailureOutput), reporter.getMessages());
166
167 reporter.reset();
168 factory.printTestFailures(TestResultType.FAILURE);
169 assertEquals(emptyList(), reporter.getMessages());
170 }
171
172 static final class DummyTestReporter implements ConsoleLogger {
173 private final List<String> messages = new ArrayList<>();
174
175 @Override
176 public boolean isDebugEnabled() {
177 return true;
178 }
179
180 @Override
181 public void debug(String message) {
182 messages.add(message);
183 }
184
185 @Override
186 public boolean isInfoEnabled() {
187 return true;
188 }
189
190 @Override
191 public void info(String message) {
192 messages.add(message);
193 }
194
195 @Override
196 public boolean isWarnEnabled() {
197 return true;
198 }
199
200 @Override
201 public void warning(String message) {
202 messages.add(message);
203 }
204
205 @Override
206 public boolean isErrorEnabled() {
207 return true;
208 }
209
210 @Override
211 public void error(String message) {
212 messages.add(message);
213 }
214
215 @Override
216 public void error(String message, Throwable t) {
217 messages.add(message + " " + t.getLocalizedMessage());
218 }
219
220 @Override
221 public void error(Throwable t) {
222 messages.add(t.getLocalizedMessage());
223 }
224
225 List<String> getMessages() {
226 return messages;
227 }
228
229 void reset() {
230 messages.clear();
231 }
232 }
233
234 public void testGetTestResultType() {
235 List<ReportEntryType> emptyList = new ArrayList<>();
236 assertEquals(TestResultType.UNKNOWN, getTestResultType(emptyList, 1));
237
238 List<ReportEntryType> successList = new ArrayList<>();
239 successList.add(ReportEntryType.SUCCESS);
240 successList.add(ReportEntryType.SUCCESS);
241 assertEquals(TestResultType.SUCCESS, getTestResultType(successList, 1));
242
243 List<ReportEntryType> failureErrorList = new ArrayList<>();
244 failureErrorList.add(ReportEntryType.FAILURE);
245 failureErrorList.add(ReportEntryType.ERROR);
246 assertEquals(TestResultType.ERROR, getTestResultType(failureErrorList, 1));
247
248 List<ReportEntryType> errorFailureList = new ArrayList<>();
249 errorFailureList.add(ReportEntryType.ERROR);
250 errorFailureList.add(ReportEntryType.FAILURE);
251 assertEquals(TestResultType.ERROR, getTestResultType(errorFailureList, 1));
252
253 List<ReportEntryType> flakeList = new ArrayList<>();
254 flakeList.add(ReportEntryType.SUCCESS);
255 flakeList.add(ReportEntryType.FAILURE);
256 assertEquals(TestResultType.FLAKE, getTestResultType(flakeList, 1));
257
258 assertEquals(TestResultType.FAILURE, getTestResultType(flakeList, 0));
259
260 flakeList = new ArrayList<>();
261 flakeList.add(ReportEntryType.ERROR);
262 flakeList.add(ReportEntryType.SUCCESS);
263 flakeList.add(ReportEntryType.FAILURE);
264 assertEquals(TestResultType.FLAKE, getTestResultType(flakeList, 1));
265
266 assertEquals(TestResultType.ERROR, getTestResultType(flakeList, 0));
267
268 List<ReportEntryType> skippedList = new ArrayList<>();
269 skippedList.add(ReportEntryType.SKIPPED);
270 assertEquals(TestResultType.SKIPPED, getTestResultType(skippedList, 1));
271 }
272
273 public void testLogger() {
274 MessageUtils.setColorEnabled(false);
275 File target = new File(System.getProperty("user.dir"), "target");
276 File reportsDirectory = new File(target, "tmp6");
277 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
278 true,
279 true,
280 "PLAIN",
281 false,
282 reportsDirectory,
283 false,
284 null,
285 new File(reportsDirectory, "TESTHASH"),
286 false,
287 1,
288 null,
289 null,
290 false,
291 new SurefireStatelessReporter(),
292 new SurefireConsoleOutputReporter(),
293 new SurefireStatelessTestsetInfoReporter());
294
295 DummyTestReporter reporter = new DummyTestReporter();
296
297 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
298
299 TestReportListener<TestOutputReportEntry> runListener = factory.createTestReportListener();
300
301 assertTrue(runListener.isDebugEnabled());
302 assertTrue(runListener.isInfoEnabled());
303 assertTrue(runListener.isWarnEnabled());
304 assertTrue(runListener.isErrorEnabled());
305
306 runListener.debug("msg");
307 assertEquals(1, reporter.getMessages().size());
308 assertEquals("msg", reporter.getMessages().get(0));
309 reporter.reset();
310
311 runListener.info("msg\n");
312 assertEquals(1, reporter.getMessages().size());
313 assertEquals("msg", reporter.getMessages().get(0));
314 reporter.reset();
315
316 runListener.warning("msg\r\n");
317 assertEquals(1, reporter.getMessages().size());
318 assertEquals("msg", reporter.getMessages().get(0));
319 reporter.reset();
320
321 runListener.error("msg");
322 assertEquals(1, reporter.getMessages().size());
323 assertEquals("msg", reporter.getMessages().get(0));
324 reporter.reset();
325
326 runListener.error("msg\n", new Exception("e"));
327 assertEquals(1, reporter.getMessages().size());
328 assertEquals("msg e", reporter.getMessages().get(0));
329 reporter.reset();
330
331 runListener.error(new Exception("e"));
332 assertEquals(1, reporter.getMessages().size());
333 assertEquals("e", reporter.getMessages().get(0));
334 reporter.reset();
335 }
336
337 public void testCreateReporterWithZeroStatistics() {
338 MessageUtils.setColorEnabled(false);
339 File target = new File(System.getProperty("user.dir"), "target");
340 File reportsDirectory = new File(target, "tmp7");
341 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
342 true,
343 true,
344 "PLAIN",
345 false,
346 reportsDirectory,
347 false,
348 null,
349 new File(reportsDirectory, "TESTHASH"),
350 false,
351 0,
352 null,
353 null,
354 false,
355 new SurefireStatelessReporter(),
356 new SurefireConsoleOutputReporter(),
357 new SurefireStatelessTestsetInfoReporter());
358
359 assertTrue(reportConfig.isUseFile());
360 assertTrue(reportConfig.isPrintSummary());
361 assertEquals("PLAIN", reportConfig.getReportFormat());
362 assertFalse(reportConfig.isRedirectTestOutputToFile());
363 assertEquals(reportsDirectory, reportConfig.getReportsDirectory());
364 assertFalse(reportConfig.isTrimStackTrace());
365 assertNull(reportConfig.getReportNameSuffix());
366 assertEquals(new File(reportsDirectory, "TESTHASH"), reportConfig.getStatisticsFile());
367 assertFalse(reportConfig.isRequiresRunHistory());
368 assertEquals(0, reportConfig.getRerunFailingTestsCount());
369 assertNull(reportConfig.getXsdSchemaLocation());
370 assertEquals(UTF_8, reportConfig.getEncoding());
371 assertFalse(reportConfig.isForking());
372 assertNotNull(reportConfig.getXmlReporter());
373 assertNotNull(reportConfig.getConsoleOutputReporter());
374 assertNotNull(reportConfig.getTestsetReporter());
375 assertNull(reportConfig.getStatisticsReporter());
376
377 DummyTestReporter reporter = new DummyTestReporter();
378
379 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
380 assertEquals(reportsDirectory, factory.getReportsDirectory());
381
382 TestSetRunListener runListener = (TestSetRunListener) factory.createTestReportListener();
383 Collection listeners = getInternalState(factory, "listeners");
384 assertEquals(1, listeners.size());
385 assertTrue(listeners.contains(runListener));
386
387 assertNotNull(runListener.getTestMethodStats());
388
389 factory.runStarting();
390
391 factory.close();
392
393 RunStatistics statistics = factory.getGlobalRunStatistics();
394 assertEquals(0, statistics.getCompletedCount());
395 assertEquals(new RunResult(0, 0, 0, 0), statistics.getRunResult());
396 assertEquals(0, statistics.getFailures());
397 assertEquals(0, statistics.getErrors());
398 assertEquals(0, statistics.getSkipped());
399 assertEquals(0, statistics.getFlakes());
400 assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", statistics.getSummary());
401 assertEquals(0, statistics.getCompletedCount());
402
403 List<String> messages = reporter.getMessages();
404 assertEquals("", messages.get(0));
405 assertEquals("-------------------------------------------------------", messages.get(1));
406 assertEquals(" T E S T S", messages.get(2));
407 assertEquals("-------------------------------------------------------", messages.get(3));
408 assertEquals("", messages.get(4));
409 assertEquals("Results:", messages.get(5));
410 assertEquals("", messages.get(6));
411 assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", messages.get(7));
412 assertEquals("", messages.get(8));
413 assertEquals(9, messages.size());
414 }
415
416 static class DummyStackTraceWriter implements StackTraceWriter {
417
418 private final String stackTrace;
419
420 DummyStackTraceWriter(String stackTrace) {
421 this.stackTrace = stackTrace;
422 }
423
424 @Override
425 public String writeTraceToString() {
426 return "";
427 }
428
429 @Override
430 public String writeTrimmedTraceToString() {
431 return "";
432 }
433
434 @Override
435 public String smartTrimmedStackTrace() {
436 return stackTrace;
437 }
438
439 @Override
440 public SafeThrowable getThrowable() {
441 return null;
442 }
443 }
444 }