1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.surefire.its;
20
21 import java.util.ArrayList;
22 import java.util.List;
23
24 import org.apache.maven.shared.verifier.VerificationException;
25 import org.apache.maven.surefire.its.fixture.OutputValidator;
26 import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
27 import org.junit.Test;
28 import org.junit.runner.RunWith;
29 import org.junit.runners.Parameterized;
30 import org.junit.runners.Parameterized.Parameter;
31 import org.junit.runners.Parameterized.Parameters;
32
33 import static java.nio.charset.StandardCharsets.UTF_8;
34 import static org.apache.maven.surefire.its.fixture.IsRegex.regex;
35 import static org.assertj.core.api.Assertions.assertThat;
36 import static org.assertj.core.util.Sets.set;
37 import static org.hamcrest.CoreMatchers.allOf;
38 import static org.hamcrest.CoreMatchers.is;
39 import static org.hamcrest.CoreMatchers.not;
40 import static org.hamcrest.CoreMatchers.startsWith;
41 import static org.hamcrest.Matchers.matchesRegex;
42 import static org.junit.Assert.assertThat;
43 import static org.junit.Assume.assumeThat;
44
45
46
47
48 @RunWith(Parameterized.class)
49 @SuppressWarnings("checkstyle:magicnumber")
50 public class JUnitPlatformEnginesIT extends SurefireJUnit4IntegrationTestCase {
51
52 private static final String XML_TESTSUITE_FRAGMENT =
53 "<testsuite xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:noNamespaceSchemaLocation="
54 + "\"https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report.xsd\" "
55 + "version=\"3.0\" name=\"<< ✨ >>\"";
56
57 @Parameter
58 @SuppressWarnings("checkstyle:visibilitymodifier")
59 public String platform;
60
61 @Parameter(1)
62 @SuppressWarnings("checkstyle:visibilitymodifier")
63 public String jupiter;
64
65 @Parameter(2)
66 @SuppressWarnings("checkstyle:visibilitymodifier")
67 public String opentest;
68
69 @Parameter(3)
70 @SuppressWarnings("checkstyle:visibilitymodifier")
71 public String apiguardian;
72
73 @Parameters(name = "{0}")
74 public static Iterable<Object[]> artifactVersions() {
75 List<Object[]> args = new ArrayList<>();
76 args.add(new Object[] {"1.8.2", "5.8.2", "1.2.0", "1.1.2"});
77 args.add(new Object[] {"1.9.1", "5.9.1", "1.2.0", "1.1.2"});
78 args.add(new Object[] {"1.10.2", "5.10.2", "1.3.0", "1.1.2"});
79 args.add(new Object[] {"1.13.4", "5.13.4", "1.3.0", "1.1.2"});
80 return args;
81 }
82
83 @Test
84 public void testToRegex() {
85 String regex = toRegex(".[]()*");
86 assertThat(regex).isEqualTo("\\.\\[\\]\\(\\).*");
87 }
88
89 @Test
90 public void platform() throws VerificationException {
91 OutputValidator validator = unpack("junit-platform", '-' + platform)
92 .sysProp("jupiter.version", jupiter)
93 .debugLogging()
94 .executeTest()
95 .verifyErrorFree(1);
96
97 List<String> lines = validator.loadLogLines(startsWith("[DEBUG] test(compact) classpath"));
98
99 assertThat(lines).hasSize(1);
100
101 String line = lines.get(0);
102
103 assertThat(
104 set(line),
105 allOf(
106 regex(toRegex("*[DEBUG] test(compact) classpath:*")),
107 regex(toRegex("* test-classes*")),
108 regex(toRegex("* classes*")),
109 regex(toRegex("*junit-jupiter-engine-" + jupiter + ".jar*")),
110 regex(toRegex("*apiguardian-api-" + apiguardian + ".jar*")),
111 regex(toRegex("*junit-platform-engine-" + platform + ".jar*")),
112 regex(toRegex("*junit-platform-commons-" + platform + ".jar*")),
113 regex(toRegex("*opentest4j-" + opentest + ".jar*")),
114 regex(toRegex("*junit-jupiter-api-" + jupiter + ".jar*"))));
115
116 lines = validator.loadLogLines(startsWith("[DEBUG] provider(compact) classpath"));
117
118 assertThat(lines).hasSize(1);
119
120 line = lines.get(0);
121
122 assertThat(
123 set(line),
124 allOf(
125 regex(toRegex("*[DEBUG] provider(compact) classpath:*")),
126 regex(toRegex("*surefire-junit-platform-*.jar*")),
127 regex(toRegex("*surefire-api-*.jar*")),
128 regex(toRegex("*surefire-logger-api-*.jar*")),
129 regex(toRegex("*common-java5-*.jar*")),
130 regex(toRegex("*junit-platform-launcher-" + platform + ".jar*"))));
131
132 lines = validator.loadLogLines(startsWith("[DEBUG] boot(compact) classpath"));
133
134 assertThat(lines).hasSize(1);
135
136 line = lines.get(0);
137
138 assertThat(
139 set(line),
140 allOf(
141 regex(toRegex("*[DEBUG] boot(compact) classpath:*")),
142 regex(toRegex("*surefire-booter-*.jar*")),
143 regex(toRegex("*surefire-api-*.jar*")),
144 regex(toRegex("*surefire-logger-api-*.jar*")),
145 regex(toRegex("* test-classes*")),
146 regex(toRegex("* classes*")),
147 regex(toRegex("*junit-jupiter-engine-" + jupiter + ".jar*")),
148 regex(toRegex("*apiguardian-api-" + apiguardian + ".jar*")),
149 regex(toRegex("*junit-platform-engine-" + platform + ".jar*")),
150 regex(toRegex("*junit-platform-commons-" + platform + ".jar*")),
151 regex(toRegex("*opentest4j-" + opentest + ".jar*")),
152 regex(toRegex("*junit-jupiter-api-" + jupiter + ".jar*")),
153 regex(toRegex("*surefire-junit-platform-*.jar*")),
154 regex(toRegex("*junit-platform-launcher-" + platform + ".jar*"))));
155 }
156
157 @Test
158 public void testJupiterEngine() {
159 unpack("junit-platform-engine-jupiter", "-" + jupiter)
160 .setTestToRun("Basic*Test")
161 .sysProp("junit5.version", jupiter)
162 .executeTest()
163 .verifyErrorFree(5);
164 }
165
166 @Test
167 public void failingBeforeAllMethod() {
168 OutputValidator validator = unpack("surefire-1688", "-" + jupiter)
169 .setTestToRun("FailingBeforeAllJupiterTest")
170 .sysProp("junit5.version", jupiter)
171 .maven()
172 .withFailure()
173 .executeTest()
174 .verifyTextInLog("oneTimeSetUp() failed")
175 .assertTestSuiteResults(1, 0, 1, 0);
176
177 validator
178 .getSurefireReportsFile("jira1688.FailingBeforeAllJupiterTest.txt", UTF_8)
179 .assertContainsText("oneTimeSetUp() failed");
180 }
181
182 @Test
183 public void errorInBeforeAllMethod() {
184 OutputValidator validator = unpack("surefire-1741", "-" + jupiter)
185 .setTestToRun("ErrorInBeforeAllJupiterTest")
186 .sysProp("junit5.version", jupiter)
187 .maven()
188 .withFailure()
189 .executeTest()
190 .verifyTextInLog("oneTimeSetUp() encountered an error")
191 .assertTestSuiteResults(1, 1, 0, 0);
192
193 validator
194 .getSurefireReportsFile("jira1741.ErrorInBeforeAllJupiterTest.txt", UTF_8)
195 .assertContainsText("oneTimeSetUp() encountered an error");
196 }
197
198 @Test
199 public void testJupiterEngineWithErrorInParameterizedSource() {
200 OutputValidator validator = unpack("surefire-1741", "-" + jupiter)
201 .setTestToRun("ErrorInParameterizedSourceJupiterTest")
202 .sysProp("junit5.version", jupiter)
203 .maven()
204 .withFailure()
205 .executeTest()
206 .verifyTextInLog("args() method source encountered an error")
207 .assertTestSuiteResults(1, 1, 0, 0);
208
209 validator
210 .getSurefireReportsFile("jira1741.ErrorInParameterizedSourceJupiterTest.txt", UTF_8)
211 .assertContainsText("args() method source encountered an error");
212 }
213
214 @Test
215 public void testJupiterEngineWithFailureInParameterizedSource() {
216 OutputValidator validator = unpack("surefire-1741", "-" + jupiter)
217 .setTestToRun("FailureInParameterizedSourceJupiterTest")
218 .sysProp("junit5.version", jupiter)
219 .maven()
220 .withFailure()
221 .executeTest()
222 .verifyTextInLog("args() method source failed")
223 .assertTestSuiteResults(1, 0, 1, 0);
224
225 validator
226 .getSurefireReportsFile("jira1741.FailureInParameterizedSourceJupiterTest.txt", UTF_8)
227 .assertContainsText("args() method source failed");
228 }
229
230 @Test
231 public void testJupiterEngineWithErrorInTestFactory() {
232 OutputValidator validator = unpack("surefire-1727", "-" + jupiter)
233 .setTestToRun("ErrorInTestFactoryJupiterTest")
234 .sysProp("junit5.version", jupiter)
235 .maven()
236 .withFailure()
237 .executeTest()
238 .verifyTextInLog("Encountered error in TestFactory testFactory()")
239 .assertTestSuiteResults(1, 1, 0, 0);
240
241 validator
242 .getSurefireReportsFile("jira1727.ErrorInTestFactoryJupiterTest.txt", UTF_8)
243 .assertContainsText("Encountered error in TestFactory testFactory()");
244 }
245
246 @Test
247 public void testJupiterEngineWithFailureInTestFactory() {
248 OutputValidator validator = unpack("surefire-1727", "-" + jupiter)
249 .setTestToRun("FailureInTestFactoryJupiterTest")
250 .sysProp("junit5.version", jupiter)
251 .maven()
252 .withFailure()
253 .executeTest()
254 .verifyTextInLog("Encountered failure in TestFactory testFactory()")
255 .assertTestSuiteResults(1, 0, 1, 0);
256
257 validator
258 .getSurefireReportsFile("jira1727.FailureInTestFactoryJupiterTest.txt", UTF_8)
259 .assertContainsText("Encountered failure in TestFactory testFactory()");
260 }
261
262 @Test
263 public void testJupiterEngineWithErrorInTestTemplateProvider() {
264 OutputValidator validator = unpack("surefire-1727", "-" + jupiter)
265 .setTestToRun("ErrorInTestTemplateProviderTest")
266 .sysProp("junit5.version", jupiter)
267 .maven()
268 .withFailure()
269 .executeTest()
270 .verifyTextInLog("Encountered error in TestTemplate provideTestTemplateInvocationContexts()")
271 .assertTestSuiteResults(1, 1, 0, 0);
272
273 validator
274 .getSurefireReportsFile("jira1727.ErrorInTestTemplateProviderTest.txt", UTF_8)
275 .assertContainsText("Encountered error in TestTemplate provideTestTemplateInvocationContexts()");
276 }
277
278 @Test
279 public void testJupiterEngineWithFailureInTestTemplateProvider() {
280 OutputValidator validator = unpack("surefire-1727", "-" + jupiter)
281 .setTestToRun("FailureInTestTemplateProviderTest")
282 .sysProp("junit5.version", jupiter)
283 .maven()
284 .withFailure()
285 .executeTest()
286 .verifyTextInLog("Encountered failure in TestTemplate provideTestTemplateInvocationContexts()")
287 .assertTestSuiteResults(1, 0, 1, 0);
288
289 validator
290 .getSurefireReportsFile("jira1727.FailureInTestTemplateProviderTest.txt", UTF_8)
291 .assertContainsText("Encountered failure in TestTemplate provideTestTemplateInvocationContexts()");
292 }
293
294 @Test
295 public void testJupiterEngineWithTestTemplateNotClassifiedAsFlake() {
296 unpack("junit5-testtemplate-bug", "-" + jupiter)
297 .setTestToRun("FieldSettingTest")
298 .sysProp("junit5.version", jupiter)
299 .maven()
300 .withFailure()
301 .executeTest()
302 .verifyTextInLog("AssertionFailedError")
303 .assertTestSuiteResults(2, 0, 1, 0, 0);
304
305 unpack("junit5-testtemplate-bug", "-" + jupiter)
306 .debugLogging()
307 .setTestToRun("FieldSettingTest")
308 .sysProp("junit5.version", jupiter)
309
310 .sysProp("surefire.rerunFailingTestsCount", "1")
311 .maven()
312 .withFailure()
313 .executeTest()
314 .verifyTextInLog("AssertionFailedError")
315 .assertTestSuiteResults(2, 0, 1, 0, 0);
316 }
317
318 @Test
319 public void testJupiterEngineWithParameterizedTestsNotClassifiedAsFlake() {
320 unpack("junit5-testtemplate-bug", "-" + jupiter)
321 .debugLogging()
322 .setTestToRun("ParamsContextTest")
323 .sysProp("junit5.version", jupiter)
324
325 .sysProp("surefire.rerunFailingTestsCount", "1")
326 .maven()
327 .withFailure()
328 .executeTest()
329 .verifyTextInLog("AssertionFailedError")
330 .assertTestSuiteResults(2, 0, 1, 0, 0);
331 }
332
333 @Test
334 public void testJupiterEngineWithAssertionsFailNoParameters() {
335
336 assumeThat(jupiter, is(not("5.0.3")));
337 assumeThat(jupiter, is(not("5.1.1")));
338
339 OutputValidator validator = unpack("surefire-1748-fail-no-parameters", "-" + jupiter)
340 .setTestToRun("AssertionsFailNoParametersJupiterTest")
341 .sysProp("junit5.version", jupiter)
342 .maven()
343 .withFailure()
344 .executeTest()
345 .verifyTextInLog("AssertionsFailNoParametersJupiterTest.doTest")
346 .assertTestSuiteResults(1, 0, 1, 0);
347
348 validator
349 .getSurefireReportsFile("jira1748.AssertionsFailNoParametersJupiterTest.txt", UTF_8)
350 .assertContainsText(
351 "jira1748.AssertionsFailNoParametersJupiterTest.doTest(AssertionsFailNoParametersJupiterTest.java");
352 }
353
354 @Test
355 public void testJupiterEngineWithAssertionsFailEmptyStringParameters() {
356 OutputValidator validator = unpack("surefire-1748", "-" + jupiter)
357 .setTestToRun("AssertionsFailEmptyStringParameterJupiterTest")
358 .sysProp("junit5.version", jupiter)
359 .maven()
360 .withFailure()
361 .executeTest()
362 .verifyTextInLog("AssertionsFailEmptyStringParameterJupiterTest.doTest")
363 .assertTestSuiteResults(1, 0, 1, 0);
364
365 validator
366 .getSurefireReportsFile("jira1748.AssertionsFailEmptyStringParameterJupiterTest.txt", UTF_8)
367 .assertContainsText(
368 "AssertionsFailEmptyStringParameterJupiterTest.doTest(AssertionsFailEmptyStringParameterJupiterTest");
369 }
370
371 @Test
372 public void testJupiterEngineWithAssertionsFailMessage() {
373 OutputValidator validator = unpack("surefire-1857-assertion-message", "-" + jupiter)
374 .setTestToRun("AssertionFailureMessageTest")
375 .sysProp("junit5.version", jupiter)
376 .maven()
377 .withFailure()
378 .executeTest()
379 .verifyTextInLog("AssertionFailureMessageTest.failedTest")
380 .assertTestSuiteResults(1, 0, 1, 0);
381
382 validator
383 .getSurefireReportsFile("TEST-jira1857.AssertionFailureMessageTest.xml", UTF_8)
384 .assertContainsText("message=\"fail_message\"");
385 }
386
387 @Test
388 public void testJupiterEngineWithExceptionMessage() {
389 OutputValidator validator = unpack("surefire-1857-exception-message", "-" + jupiter)
390 .setTestToRun("ExceptionMessageTest")
391 .sysProp("junit5.version", jupiter)
392 .maven()
393 .withFailure()
394 .executeTest()
395 .verifyTextInLog("ExceptionMessageTest.errorTest")
396 .assertTestSuiteResults(1, 1, 0, 0);
397
398 validator
399 .getSurefireReportsFile("TEST-jira1857.ExceptionMessageTest.xml", UTF_8)
400 .assertContainsText("message=\"error_message\"");
401 }
402
403 @Test
404 public void testJupiterEngineWithDisplayNames() throws VerificationException {
405 OutputValidator validator = unpack("junit-platform-engine-jupiter", "-" + jupiter)
406 .sysProp("junit5.version", jupiter)
407 .executeTest()
408 .verifyErrorFree(7);
409
410 validator
411 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest.txt", UTF_8)
412 .assertContainsText("<< ✨ >>");
413
414 validator
415 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest.txt", UTF_8)
416 .assertContainsText("Test set: << ✨ >>");
417
418 validator
419 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest.txt", UTF_8)
420 .assertContainsText(" -- in << ✨ >>");
421
422 validator
423 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest-output.txt", UTF_8)
424 .assertContainsText("<< ✨ >>");
425
426 validator
427 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest-output.txt", UTF_8)
428 .assertContainsText("73$71 ✔");
429
430 validator
431 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest-output.txt", UTF_8)
432 .assertContainsText("73$72 ✔");
433
434 validator
435 .getSurefireReportsFile("TEST-junitplatformenginejupiter.DisplayNameTest.xml", UTF_8)
436 .assertContainsText("testcase name=\"73$71 ✔\" classname=\"<< ✨ >>\"")
437 .assertContainsText("testcase name=\"73$72 ✔\" classname=\"<< ✨ >>\"")
438 .assertContainsText(XML_TESTSUITE_FRAGMENT);
439
440 validator
441 .getSurefireReportsFile("TEST-junitplatformenginejupiter.BasicJupiterTest.xml", UTF_8)
442 .assertContainsText("<testcase name=\"test(TestInfo)\" "
443 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"")
444 .assertContainsText("<testcase name=\"add(int, int, int) 0 + 1 = 1\" "
445 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"")
446 .assertContainsText("<testcase name=\"add(int, int, int) 1 + 2 = 3\" "
447 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"")
448 .assertContainsText("<testcase name=\"add(int, int, int) 49 + 51 = 100\" "
449 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"")
450 .assertContainsText("<testcase name=\"add(int, int, int) 1 + 100 = 101\" "
451 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"");
452
453 validator
454 .assertThatLogLine(matchesRegex(".*Running junitplatformenginejupiter.BasicJupiterTest$"), is(1))
455 .assertThatLogLine(matchesRegex(".*Tests run.* junitplatformenginejupiter.BasicJupiterTest$"), is(1))
456 .assertThatLogLine(matchesRegex(".*Running << . >>$"), is(1))
457 .assertThatLogLine(matchesRegex(".*Tests run.* << . >>$"), is(1));
458 }
459
460 @Test
461 public void testTags() {
462
463
464 assumeThat(jupiter, is(not("5.0.3")));
465
466 unpack("junit-platform-tags", "-" + jupiter)
467 .sysProp("junit5.version", jupiter)
468 .executeTest()
469 .verifyErrorFree(2);
470 }
471
472 private static String toRegex(String text) {
473 return text.replaceAll("\\.", "\\\\.")
474 .replaceAll("\\[", "\\\\[")
475 .replaceAll("]", "\\\\]")
476 .replaceAll("\\(", "\\\\(")
477 .replaceAll("\\)", "\\\\)")
478 .replaceAll("\\*", ".*");
479 }
480 }