1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.plugin.surefire.report;
20
21 import java.io.File;
22 import java.util.ArrayDeque;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.Queue;
27
28 import junit.framework.TestCase;
29 import org.apache.maven.plugin.surefire.StartupReportConfiguration;
30 import org.apache.maven.plugin.surefire.extensions.SurefireConsoleOutputReporter;
31 import org.apache.maven.plugin.surefire.extensions.SurefireStatelessReporter;
32 import org.apache.maven.plugin.surefire.extensions.SurefireStatelessTestsetInfoReporter;
33 import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
34 import org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
35 import org.apache.maven.surefire.api.report.ReporterFactoryOptions;
36 import org.apache.maven.surefire.api.report.SafeThrowable;
37 import org.apache.maven.surefire.api.report.StackTraceWriter;
38 import org.apache.maven.surefire.api.report.TestOutputReportEntry;
39 import org.apache.maven.surefire.api.report.TestReportListener;
40 import org.apache.maven.surefire.api.suite.RunResult;
41 import org.apache.maven.surefire.report.RunStatistics;
42 import org.apache.maven.surefire.shared.utils.logging.MessageUtils;
43
44 import static java.nio.charset.StandardCharsets.UTF_8;
45 import static java.util.Arrays.asList;
46 import static java.util.Collections.emptyList;
47 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.getTestResultType;
48 import static org.mockito.Mockito.mock;
49 import static org.mockito.Mockito.when;
50 import static org.powermock.reflect.Whitebox.getInternalState;
51 import static org.powermock.reflect.Whitebox.invokeMethod;
52
53
54
55
56 public class DefaultReporterFactoryTest extends TestCase {
57 private static final String TEST_ONE = "testOne";
58
59 private static final String TEST_TWO = "testTwo";
60
61 private static final String TEST_THREE = "testThree";
62
63 private static final String TEST_FOUR = "testFour";
64
65 private static final String TEST_FIVE = "testFive";
66
67 private static final String ASSERTION_FAIL = "assertionFail";
68
69 private static final String ERROR = "error";
70
71 private static final String TEST_BEFORE_ALL_FLAKE = "com.example.FlakyClass";
72
73 private static final String TEST_BEFORE_ALL_ERROR = "com.example.AlwaysFailClass";
74
75 private static final String TEST_ERROR_SUFFIX = "-- Time elapsed: 292.2 s <<< ERROR!";
76
77 public void testMergeTestHistoryResult() throws Exception {
78 MessageUtils.setColorEnabled(false);
79 File target = new File(System.getProperty("user.dir"), "target");
80 File reportsDirectory = new File(target, "tmp5");
81 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
82 true,
83 true,
84 "PLAIN",
85 false,
86 reportsDirectory,
87 false,
88 null,
89 new File(reportsDirectory, "TESTHASH"),
90 false,
91 1,
92 null,
93 null,
94 false,
95 true,
96 true,
97 new SurefireStatelessReporter(),
98 new SurefireConsoleOutputReporter(),
99 new SurefireStatelessTestsetInfoReporter(),
100 new ReporterFactoryOptions());
101
102 DummyTestReporter reporter = new DummyTestReporter();
103
104 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
105
106
107 Queue<TestMethodStats> firstRunStats = new ArrayDeque<>();
108 firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
109 firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
110 firstRunStats.add(
111 new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
112 firstRunStats.add(
113 new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
114 firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null));
115
116 firstRunStats.add(new TestMethodStats(
117 TEST_BEFORE_ALL_FLAKE + ".null",
118 ReportEntryType.ERROR,
119 new DummyStackTraceWriter(TEST_BEFORE_ALL_FLAKE + ".null " + TEST_ERROR_SUFFIX)));
120
121
122 Queue<TestMethodStats> secondRunStats = new ArrayDeque<>();
123 secondRunStats.add(
124 new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
125 secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null));
126 secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
127 secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null));
128
129 secondRunStats.add(new TestMethodStats(TEST_BEFORE_ALL_FLAKE + ".testSucceed", ReportEntryType.SUCCESS, null));
130
131 secondRunStats.add(new TestMethodStats(
132 TEST_BEFORE_ALL_ERROR + ".null",
133 ReportEntryType.ERROR,
134 new DummyStackTraceWriter(TEST_BEFORE_ALL_ERROR + ".null " + TEST_ERROR_SUFFIX)));
135
136
137 Queue<TestMethodStats> thirdRunStats = new ArrayDeque<>();
138 thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null));
139 thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
140
141 thirdRunStats.add(new TestMethodStats(
142 TEST_BEFORE_ALL_ERROR + ".null",
143 ReportEntryType.ERROR,
144 new DummyStackTraceWriter(TEST_BEFORE_ALL_ERROR + ".null")));
145
146 TestSetRunListener firstRunListener = mock(TestSetRunListener.class);
147 TestSetRunListener secondRunListener = mock(TestSetRunListener.class);
148 TestSetRunListener thirdRunListener = mock(TestSetRunListener.class);
149 when(firstRunListener.getTestMethodStats()).thenReturn(firstRunStats);
150 when(secondRunListener.getTestMethodStats()).thenReturn(secondRunStats);
151 when(thirdRunListener.getTestMethodStats()).thenReturn(thirdRunStats);
152
153 factory.addListener(firstRunListener);
154 factory.addListener(secondRunListener);
155 factory.addListener(thirdRunListener);
156
157 invokeMethod(factory, "mergeTestHistoryResult");
158 RunStatistics mergedStatistics = factory.getGlobalRunStatistics();
159
160
161 assertEquals(7, mergedStatistics.getCompletedCount());
162 assertEquals(2, mergedStatistics.getErrors());
163 assertEquals(0, mergedStatistics.getFailures());
164 assertEquals(4, mergedStatistics.getFlakes());
165 assertEquals(0, mergedStatistics.getSkipped());
166
167
168 factory.printTestFailures(TestResultType.FLAKE);
169 String[] expectedFlakeOutput = {
170 "Flakes: ",
171 TEST_BEFORE_ALL_FLAKE + ".<beforeAll>",
172 " Run 1: " + TEST_BEFORE_ALL_FLAKE + ".null " + TEST_ERROR_SUFFIX,
173 " Run 2: PASS",
174 "",
175 TEST_FOUR,
176 " Run 1: " + ASSERTION_FAIL,
177 " Run 2: PASS",
178 "",
179 TEST_ONE,
180 " Run 1: " + ERROR,
181 " Run 2: " + ASSERTION_FAIL,
182 " Run 3: PASS",
183 "",
184 TEST_TWO,
185 " Run 1: " + ERROR,
186 " Run 2: PASS",
187 ""
188 };
189 assertEquals(asList(expectedFlakeOutput), reporter.getMessages());
190
191 reporter.reset();
192 factory.printTestFailures(TestResultType.ERROR);
193 String[] expectedFailureOutput = {
194 "Errors: ",
195 TEST_BEFORE_ALL_ERROR + ".<beforeAll>",
196 " Run 1: " + TEST_BEFORE_ALL_ERROR + ".null " + TEST_ERROR_SUFFIX,
197 " Run 2: " + TEST_BEFORE_ALL_ERROR + ".null",
198 "",
199 TEST_THREE,
200 " Run 1: " + ASSERTION_FAIL,
201 " Run 2: " + ERROR,
202 " Run 3: " + ERROR,
203 ""
204 };
205 assertEquals(asList(expectedFailureOutput), reporter.getMessages());
206
207 reporter.reset();
208 factory.printTestFailures(TestResultType.FAILURE);
209 assertEquals(emptyList(), reporter.getMessages());
210 }
211
212 static final class DummyTestReporter implements ConsoleLogger {
213 private final List<String> messages = new ArrayList<>();
214
215 @Override
216 public boolean isDebugEnabled() {
217 return true;
218 }
219
220 @Override
221 public void debug(String message) {
222 messages.add(message);
223 }
224
225 @Override
226 public boolean isInfoEnabled() {
227 return true;
228 }
229
230 @Override
231 public void info(String message) {
232 messages.add(message);
233 }
234
235 @Override
236 public boolean isWarnEnabled() {
237 return true;
238 }
239
240 @Override
241 public void warning(String message) {
242 messages.add(message);
243 }
244
245 @Override
246 public boolean isErrorEnabled() {
247 return true;
248 }
249
250 @Override
251 public void error(String message) {
252 messages.add(message);
253 }
254
255 @Override
256 public void error(String message, Throwable t) {
257 messages.add(message + " " + t.getLocalizedMessage());
258 }
259
260 @Override
261 public void error(Throwable t) {
262 messages.add(t.getLocalizedMessage());
263 }
264
265 List<String> getMessages() {
266 return messages;
267 }
268
269 void reset() {
270 messages.clear();
271 }
272 }
273
274 public void testGetTestResultType() {
275 List<ReportEntryType> emptyList = new ArrayList<>();
276 assertEquals(TestResultType.UNKNOWN, getTestResultType(emptyList, 1));
277
278 List<ReportEntryType> successList = new ArrayList<>();
279 successList.add(ReportEntryType.SUCCESS);
280 successList.add(ReportEntryType.SUCCESS);
281 assertEquals(TestResultType.SUCCESS, getTestResultType(successList, 1));
282
283 List<ReportEntryType> failureErrorList = new ArrayList<>();
284 failureErrorList.add(ReportEntryType.FAILURE);
285 failureErrorList.add(ReportEntryType.ERROR);
286 assertEquals(TestResultType.ERROR, getTestResultType(failureErrorList, 1));
287
288 List<ReportEntryType> errorFailureList = new ArrayList<>();
289 errorFailureList.add(ReportEntryType.ERROR);
290 errorFailureList.add(ReportEntryType.FAILURE);
291 assertEquals(TestResultType.ERROR, getTestResultType(errorFailureList, 1));
292
293 List<ReportEntryType> flakeList = new ArrayList<>();
294 flakeList.add(ReportEntryType.SUCCESS);
295 flakeList.add(ReportEntryType.FAILURE);
296 assertEquals(TestResultType.FLAKE, getTestResultType(flakeList, 1));
297
298 assertEquals(TestResultType.FAILURE, getTestResultType(flakeList, 0));
299
300 flakeList = new ArrayList<>();
301 flakeList.add(ReportEntryType.ERROR);
302 flakeList.add(ReportEntryType.SUCCESS);
303 flakeList.add(ReportEntryType.FAILURE);
304 assertEquals(TestResultType.FLAKE, getTestResultType(flakeList, 1));
305
306 assertEquals(TestResultType.ERROR, getTestResultType(flakeList, 0));
307
308 List<ReportEntryType> skippedList = new ArrayList<>();
309 skippedList.add(ReportEntryType.SKIPPED);
310 assertEquals(TestResultType.SKIPPED, getTestResultType(skippedList, 1));
311 }
312
313 public void testLogger() {
314 MessageUtils.setColorEnabled(false);
315 File target = new File(System.getProperty("user.dir"), "target");
316 File reportsDirectory = new File(target, "tmp6");
317 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
318 true,
319 true,
320 "PLAIN",
321 false,
322 reportsDirectory,
323 false,
324 null,
325 new File(reportsDirectory, "TESTHASH"),
326 false,
327 1,
328 null,
329 null,
330 false,
331 true,
332 true,
333 new SurefireStatelessReporter(),
334 new SurefireConsoleOutputReporter(),
335 new SurefireStatelessTestsetInfoReporter(),
336 new ReporterFactoryOptions());
337
338 DummyTestReporter reporter = new DummyTestReporter();
339
340 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
341
342 TestReportListener<TestOutputReportEntry> runListener = factory.createTestReportListener();
343
344 assertTrue(runListener.isDebugEnabled());
345 assertTrue(runListener.isInfoEnabled());
346 assertTrue(runListener.isWarnEnabled());
347 assertTrue(runListener.isErrorEnabled());
348
349 runListener.debug("msg");
350 assertEquals(1, reporter.getMessages().size());
351 assertEquals("msg", reporter.getMessages().get(0));
352 reporter.reset();
353
354 runListener.info("msg\n");
355 assertEquals(1, reporter.getMessages().size());
356 assertEquals("msg", reporter.getMessages().get(0));
357 reporter.reset();
358
359 runListener.warning("msg\r\n");
360 assertEquals(1, reporter.getMessages().size());
361 assertEquals("msg", reporter.getMessages().get(0));
362 reporter.reset();
363
364 runListener.error("msg");
365 assertEquals(1, reporter.getMessages().size());
366 assertEquals("msg", reporter.getMessages().get(0));
367 reporter.reset();
368
369 runListener.error("msg\n", new Exception("e"));
370 assertEquals(1, reporter.getMessages().size());
371 assertEquals("msg e", reporter.getMessages().get(0));
372 reporter.reset();
373
374 runListener.error(new Exception("e"));
375 assertEquals(1, reporter.getMessages().size());
376 assertEquals("e", reporter.getMessages().get(0));
377 reporter.reset();
378 }
379
380 public void testCreateReporterWithZeroStatistics() {
381 MessageUtils.setColorEnabled(false);
382 File target = new File(System.getProperty("user.dir"), "target");
383 File reportsDirectory = new File(target, "tmp7");
384 StartupReportConfiguration reportConfig = new StartupReportConfiguration(
385 true,
386 true,
387 "PLAIN",
388 false,
389 reportsDirectory,
390 false,
391 null,
392 new File(reportsDirectory, "TESTHASH"),
393 false,
394 0,
395 null,
396 null,
397 false,
398 true,
399 true,
400 new SurefireStatelessReporter(),
401 new SurefireConsoleOutputReporter(),
402 new SurefireStatelessTestsetInfoReporter(),
403 new ReporterFactoryOptions());
404
405 assertTrue(reportConfig.isUseFile());
406 assertTrue(reportConfig.isPrintSummary());
407 assertEquals("PLAIN", reportConfig.getReportFormat());
408 assertFalse(reportConfig.isRedirectTestOutputToFile());
409 assertEquals(reportsDirectory, reportConfig.getReportsDirectory());
410 assertFalse(reportConfig.isTrimStackTrace());
411 assertNull(reportConfig.getReportNameSuffix());
412 assertEquals(new File(reportsDirectory, "TESTHASH"), reportConfig.getStatisticsFile());
413 assertFalse(reportConfig.isRequiresRunHistory());
414 assertEquals(0, reportConfig.getRerunFailingTestsCount());
415 assertNull(reportConfig.getXsdSchemaLocation());
416 assertEquals(UTF_8, reportConfig.getEncoding());
417 assertFalse(reportConfig.isForking());
418 assertNotNull(reportConfig.getXmlReporter());
419 assertNotNull(reportConfig.getConsoleOutputReporter());
420 assertNotNull(reportConfig.getTestsetReporter());
421 assertNull(reportConfig.getStatisticsReporter());
422
423 DummyTestReporter reporter = new DummyTestReporter();
424
425 DefaultReporterFactory factory = new DefaultReporterFactory(reportConfig, reporter);
426 assertEquals(reportsDirectory, factory.getReportsDirectory());
427
428 TestSetRunListener runListener = (TestSetRunListener) factory.createTestReportListener();
429 Collection listeners = getInternalState(factory, "listeners");
430 assertEquals(1, listeners.size());
431 assertTrue(listeners.contains(runListener));
432
433 assertNotNull(runListener.getTestMethodStats());
434
435 factory.runStarting();
436
437 factory.close();
438
439 RunStatistics statistics = factory.getGlobalRunStatistics();
440 assertEquals(0, statistics.getCompletedCount());
441 assertEquals(new RunResult(0, 0, 0, 0), statistics.getRunResult());
442 assertEquals(0, statistics.getFailures());
443 assertEquals(0, statistics.getErrors());
444 assertEquals(0, statistics.getSkipped());
445 assertEquals(0, statistics.getFlakes());
446 assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", statistics.getSummary());
447 assertEquals(0, statistics.getCompletedCount());
448
449 List<String> messages = reporter.getMessages();
450 assertEquals("", messages.get(0));
451 assertEquals("-------------------------------------------------------", messages.get(1));
452 assertEquals(" T E S T S", messages.get(2));
453 assertEquals("-------------------------------------------------------", messages.get(3));
454 assertEquals("", messages.get(4));
455 assertEquals("Results:", messages.get(5));
456 assertEquals("", messages.get(6));
457 assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", messages.get(7));
458 assertEquals("", messages.get(8));
459 assertEquals(9, messages.size());
460 }
461
462 static class DummyStackTraceWriter implements StackTraceWriter {
463
464 private final String stackTrace;
465
466 DummyStackTraceWriter(String stackTrace) {
467 this.stackTrace = stackTrace;
468 }
469
470 @Override
471 public String writeTraceToString() {
472 return "";
473 }
474
475 @Override
476 public String writeTrimmedTraceToString() {
477 return "";
478 }
479
480 @Override
481 public String smartTrimmedStackTrace() {
482 return stackTrace;
483 }
484
485 @Override
486 public SafeThrowable getThrowable() {
487 return null;
488 }
489 }
490 }