1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.surefire.its;
20
21 import java.util.ArrayList;
22 import java.util.List;
23
24 import org.apache.maven.shared.verifier.VerificationException;
25 import org.apache.maven.surefire.its.fixture.OutputValidator;
26 import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
27 import org.junit.Test;
28 import org.junit.runner.RunWith;
29 import org.junit.runners.Parameterized;
30 import org.junit.runners.Parameterized.Parameter;
31 import org.junit.runners.Parameterized.Parameters;
32
33 import static java.nio.charset.StandardCharsets.UTF_8;
34 import static org.apache.maven.surefire.its.fixture.IsRegex.regex;
35 import static org.assertj.core.api.Assertions.assertThat;
36 import static org.assertj.core.util.Sets.set;
37 import static org.hamcrest.CoreMatchers.allOf;
38 import static org.hamcrest.CoreMatchers.is;
39 import static org.hamcrest.CoreMatchers.not;
40 import static org.hamcrest.CoreMatchers.startsWith;
41 import static org.hamcrest.Matchers.matchesRegex;
42 import static org.junit.Assert.assertThat;
43 import static org.junit.Assume.assumeThat;
44
45
46
47
48 @RunWith(Parameterized.class)
49 @SuppressWarnings("checkstyle:magicnumber")
50 public class JUnitPlatformEnginesIT extends SurefireJUnit4IntegrationTestCase {
51
52 private static final String XML_TESTSUITE_FRAGMENT =
53 "<testsuite xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:noNamespaceSchemaLocation="
54 + "\"https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report.xsd\" "
55 + "version=\"3.0\" name=\"<< ✨ >>\"";
56
57 @Parameter
58 @SuppressWarnings("checkstyle:visibilitymodifier")
59 public String platform;
60
61 @Parameter(1)
62 @SuppressWarnings("checkstyle:visibilitymodifier")
63 public String jupiter;
64
65 @Parameter(2)
66 @SuppressWarnings("checkstyle:visibilitymodifier")
67 public String opentest;
68
69 @Parameter(3)
70 @SuppressWarnings("checkstyle:visibilitymodifier")
71 public String apiguardian;
72
73 @Parameters(name = "{0}")
74 public static Iterable<Object[]> artifactVersions() {
75 List<Object[]> args = new ArrayList<>();
76 args.add(new Object[] {"1.2.0", "5.2.0", "1.1.0", "1.0.0"});
77 args.add(new Object[] {"1.3.2", "5.3.2", "1.1.1", "1.0.0"});
78 args.add(new Object[] {"1.4.2", "5.4.2", "1.1.1", "1.0.0"});
79 args.add(new Object[] {"1.5.2", "5.5.2", "1.2.0", "1.1.0"});
80 args.add(new Object[] {"1.6.3", "5.6.3", "1.2.0", "1.1.0"});
81 args.add(new Object[] {"1.7.2", "5.7.2", "1.2.0", "1.1.0"});
82 args.add(new Object[] {"1.8.2", "5.8.2", "1.2.0", "1.1.2"});
83 args.add(new Object[] {"1.9.1", "5.9.1", "1.2.0", "1.1.2"});
84 args.add(new Object[] {"1.10.2", "5.10.2", "1.3.0", "1.1.2"});
85 return args;
86 }
87
88 @Test
89 public void testToRegex() {
90 String regex = toRegex(".[]()*");
91 assertThat(regex).isEqualTo("\\.\\[\\]\\(\\).*");
92 }
93
94 @Test
95 public void platform() throws VerificationException {
96 OutputValidator validator = unpack("junit-platform", '-' + platform)
97 .sysProp("jupiter.version", jupiter)
98 .debugLogging()
99 .executeTest()
100 .verifyErrorFree(1);
101
102 List<String> lines = validator.loadLogLines(startsWith("[DEBUG] test(compact) classpath"));
103
104 assertThat(lines).hasSize(1);
105
106 String line = lines.get(0);
107
108 assertThat(
109 set(line),
110 allOf(
111 regex(toRegex("*[DEBUG] test(compact) classpath:*")),
112 regex(toRegex("* test-classes*")),
113 regex(toRegex("* classes*")),
114 regex(toRegex("*junit-jupiter-engine-" + jupiter + ".jar*")),
115 regex(toRegex("*apiguardian-api-" + apiguardian + ".jar*")),
116 regex(toRegex("*junit-platform-engine-" + platform + ".jar*")),
117 regex(toRegex("*junit-platform-commons-" + platform + ".jar*")),
118 regex(toRegex("*opentest4j-" + opentest + ".jar*")),
119 regex(toRegex("*junit-jupiter-api-" + jupiter + ".jar*"))));
120
121 lines = validator.loadLogLines(startsWith("[DEBUG] provider(compact) classpath"));
122
123 assertThat(lines).hasSize(1);
124
125 line = lines.get(0);
126
127 assertThat(
128 set(line),
129 allOf(
130 regex(toRegex("*[DEBUG] provider(compact) classpath:*")),
131 regex(toRegex("*surefire-junit-platform-*.jar*")),
132 regex(toRegex("*surefire-api-*.jar*")),
133 regex(toRegex("*surefire-logger-api-*.jar*")),
134 regex(toRegex("*common-java5-*.jar*")),
135 regex(toRegex("*junit-platform-launcher-" + platform + ".jar*"))));
136
137 lines = validator.loadLogLines(startsWith("[DEBUG] boot(compact) classpath"));
138
139 assertThat(lines).hasSize(1);
140
141 line = lines.get(0);
142
143 assertThat(
144 set(line),
145 allOf(
146 regex(toRegex("*[DEBUG] boot(compact) classpath:*")),
147 regex(toRegex("*surefire-booter-*.jar*")),
148 regex(toRegex("*surefire-api-*.jar*")),
149 regex(toRegex("*surefire-logger-api-*.jar*")),
150 regex(toRegex("* test-classes*")),
151 regex(toRegex("* classes*")),
152 regex(toRegex("*junit-jupiter-engine-" + jupiter + ".jar*")),
153 regex(toRegex("*apiguardian-api-" + apiguardian + ".jar*")),
154 regex(toRegex("*junit-platform-engine-" + platform + ".jar*")),
155 regex(toRegex("*junit-platform-commons-" + platform + ".jar*")),
156 regex(toRegex("*opentest4j-" + opentest + ".jar*")),
157 regex(toRegex("*junit-jupiter-api-" + jupiter + ".jar*")),
158 regex(toRegex("*surefire-junit-platform-*.jar*")),
159 regex(toRegex("*junit-platform-launcher-" + platform + ".jar*"))));
160 }
161
162 @Test
163 public void testJupiterEngine() {
164 unpack("junit-platform-engine-jupiter", "-" + jupiter)
165 .setTestToRun("Basic*Test")
166 .sysProp("junit5.version", jupiter)
167 .executeTest()
168 .verifyErrorFree(5);
169 }
170
171 @Test
172 public void failingBeforeAllMethod() {
173 OutputValidator validator = unpack("surefire-1688", "-" + jupiter)
174 .setTestToRun("FailingBeforeAllJupiterTest")
175 .sysProp("junit5.version", jupiter)
176 .maven()
177 .withFailure()
178 .executeTest()
179 .verifyTextInLog("oneTimeSetUp() failed")
180 .assertTestSuiteResults(1, 0, 1, 0);
181
182 validator
183 .getSurefireReportsFile("jira1688.FailingBeforeAllJupiterTest.txt", UTF_8)
184 .assertContainsText("oneTimeSetUp() failed");
185 }
186
187 @Test
188 public void errorInBeforeAllMethod() {
189 OutputValidator validator = unpack("surefire-1741", "-" + jupiter)
190 .setTestToRun("ErrorInBeforeAllJupiterTest")
191 .sysProp("junit5.version", jupiter)
192 .maven()
193 .withFailure()
194 .executeTest()
195 .verifyTextInLog("oneTimeSetUp() encountered an error")
196 .assertTestSuiteResults(1, 1, 0, 0);
197
198 validator
199 .getSurefireReportsFile("jira1741.ErrorInBeforeAllJupiterTest.txt", UTF_8)
200 .assertContainsText("oneTimeSetUp() encountered an error");
201 }
202
203 @Test
204 public void testJupiterEngineWithErrorInParameterizedSource() {
205 OutputValidator validator = unpack("surefire-1741", "-" + jupiter)
206 .setTestToRun("ErrorInParameterizedSourceJupiterTest")
207 .sysProp("junit5.version", jupiter)
208 .maven()
209 .withFailure()
210 .executeTest()
211 .verifyTextInLog("args() method source encountered an error")
212 .assertTestSuiteResults(1, 1, 0, 0);
213
214 validator
215 .getSurefireReportsFile("jira1741.ErrorInParameterizedSourceJupiterTest.txt", UTF_8)
216 .assertContainsText("args() method source encountered an error");
217 }
218
219 @Test
220 public void testJupiterEngineWithFailureInParameterizedSource() {
221 OutputValidator validator = unpack("surefire-1741", "-" + jupiter)
222 .setTestToRun("FailureInParameterizedSourceJupiterTest")
223 .sysProp("junit5.version", jupiter)
224 .maven()
225 .withFailure()
226 .executeTest()
227 .verifyTextInLog("args() method source failed")
228 .assertTestSuiteResults(1, 0, 1, 0);
229
230 validator
231 .getSurefireReportsFile("jira1741.FailureInParameterizedSourceJupiterTest.txt", UTF_8)
232 .assertContainsText("args() method source failed");
233 }
234
235 @Test
236 public void testJupiterEngineWithErrorInTestFactory() {
237 OutputValidator validator = unpack("surefire-1727", "-" + jupiter)
238 .setTestToRun("ErrorInTestFactoryJupiterTest")
239 .sysProp("junit5.version", jupiter)
240 .maven()
241 .withFailure()
242 .executeTest()
243 .verifyTextInLog("Encountered error in TestFactory testFactory()")
244 .assertTestSuiteResults(1, 1, 0, 0);
245
246 validator
247 .getSurefireReportsFile("jira1727.ErrorInTestFactoryJupiterTest.txt", UTF_8)
248 .assertContainsText("Encountered error in TestFactory testFactory()");
249 }
250
251 @Test
252 public void testJupiterEngineWithFailureInTestFactory() {
253 OutputValidator validator = unpack("surefire-1727", "-" + jupiter)
254 .setTestToRun("FailureInTestFactoryJupiterTest")
255 .sysProp("junit5.version", jupiter)
256 .maven()
257 .withFailure()
258 .executeTest()
259 .verifyTextInLog("Encountered failure in TestFactory testFactory()")
260 .assertTestSuiteResults(1, 0, 1, 0);
261
262 validator
263 .getSurefireReportsFile("jira1727.FailureInTestFactoryJupiterTest.txt", UTF_8)
264 .assertContainsText("Encountered failure in TestFactory testFactory()");
265 }
266
267 @Test
268 public void testJupiterEngineWithErrorInTestTemplateProvider() {
269 OutputValidator validator = unpack("surefire-1727", "-" + jupiter)
270 .setTestToRun("ErrorInTestTemplateProviderTest")
271 .sysProp("junit5.version", jupiter)
272 .maven()
273 .withFailure()
274 .executeTest()
275 .verifyTextInLog("Encountered error in TestTemplate provideTestTemplateInvocationContexts()")
276 .assertTestSuiteResults(1, 1, 0, 0);
277
278 validator
279 .getSurefireReportsFile("jira1727.ErrorInTestTemplateProviderTest.txt", UTF_8)
280 .assertContainsText("Encountered error in TestTemplate provideTestTemplateInvocationContexts()");
281 }
282
283 @Test
284 public void testJupiterEngineWithFailureInTestTemplateProvider() {
285 OutputValidator validator = unpack("surefire-1727", "-" + jupiter)
286 .setTestToRun("FailureInTestTemplateProviderTest")
287 .sysProp("junit5.version", jupiter)
288 .maven()
289 .withFailure()
290 .executeTest()
291 .verifyTextInLog("Encountered failure in TestTemplate provideTestTemplateInvocationContexts()")
292 .assertTestSuiteResults(1, 0, 1, 0);
293
294 validator
295 .getSurefireReportsFile("jira1727.FailureInTestTemplateProviderTest.txt", UTF_8)
296 .assertContainsText("Encountered failure in TestTemplate provideTestTemplateInvocationContexts()");
297 }
298
299 @Test
300 public void testJupiterEngineWithTestTemplateNotClassifiedAsFlake() {
301 unpack("junit5-testtemplate-bug", "-" + jupiter)
302 .setTestToRun("FieldSettingTest")
303 .sysProp("junit5.version", jupiter)
304 .maven()
305 .withFailure()
306 .executeTest()
307 .verifyTextInLog("AssertionFailedError")
308 .assertTestSuiteResults(2, 0, 1, 0, 0);
309
310 unpack("junit5-testtemplate-bug", "-" + jupiter)
311 .debugLogging()
312 .setTestToRun("FieldSettingTest")
313 .sysProp("junit5.version", jupiter)
314
315 .sysProp("surefire.rerunFailingTestsCount", "1")
316 .maven()
317 .withFailure()
318 .executeTest()
319 .verifyTextInLog("AssertionFailedError")
320 .assertTestSuiteResults(2, 0, 1, 0, 0);
321 }
322
323 @Test
324 public void testJupiterEngineWithParameterizedTestsNotClassifiedAsFlake() {
325 unpack("junit5-testtemplate-bug", "-" + jupiter)
326 .debugLogging()
327 .setTestToRun("ParamsContextTest")
328 .sysProp("junit5.version", jupiter)
329
330 .sysProp("surefire.rerunFailingTestsCount", "1")
331 .maven()
332 .withFailure()
333 .executeTest()
334 .verifyTextInLog("AssertionFailedError")
335 .assertTestSuiteResults(2, 0, 1, 0, 0);
336 }
337
338 @Test
339 public void testJupiterEngineWithAssertionsFailNoParameters() {
340
341 assumeThat(jupiter, is(not("5.0.3")));
342 assumeThat(jupiter, is(not("5.1.1")));
343
344 OutputValidator validator = unpack("surefire-1748-fail-no-parameters", "-" + jupiter)
345 .setTestToRun("AssertionsFailNoParametersJupiterTest")
346 .sysProp("junit5.version", jupiter)
347 .maven()
348 .withFailure()
349 .executeTest()
350 .verifyTextInLog("AssertionsFailNoParametersJupiterTest.doTest")
351 .assertTestSuiteResults(1, 0, 1, 0);
352
353 validator
354 .getSurefireReportsFile("jira1748.AssertionsFailNoParametersJupiterTest.txt", UTF_8)
355 .assertContainsText(
356 "jira1748.AssertionsFailNoParametersJupiterTest.doTest(AssertionsFailNoParametersJupiterTest.java");
357 }
358
359 @Test
360 public void testJupiterEngineWithAssertionsFailEmptyStringParameters() {
361 OutputValidator validator = unpack("surefire-1748", "-" + jupiter)
362 .setTestToRun("AssertionsFailEmptyStringParameterJupiterTest")
363 .sysProp("junit5.version", jupiter)
364 .maven()
365 .withFailure()
366 .executeTest()
367 .verifyTextInLog("AssertionsFailEmptyStringParameterJupiterTest.doTest")
368 .assertTestSuiteResults(1, 0, 1, 0);
369
370 validator
371 .getSurefireReportsFile("jira1748.AssertionsFailEmptyStringParameterJupiterTest.txt", UTF_8)
372 .assertContainsText(
373 "AssertionsFailEmptyStringParameterJupiterTest.doTest(AssertionsFailEmptyStringParameterJupiterTest");
374 }
375
376 @Test
377 public void testJupiterEngineWithAssertionsFailMessage() {
378 OutputValidator validator = unpack("surefire-1857-assertion-message", "-" + jupiter)
379 .setTestToRun("AssertionFailureMessageTest")
380 .sysProp("junit5.version", jupiter)
381 .maven()
382 .withFailure()
383 .executeTest()
384 .verifyTextInLog("AssertionFailureMessageTest.failedTest")
385 .assertTestSuiteResults(1, 0, 1, 0);
386
387 validator
388 .getSurefireReportsFile("TEST-jira1857.AssertionFailureMessageTest.xml", UTF_8)
389 .assertContainsText("message=\"fail_message\"");
390 }
391
392 @Test
393 public void testJupiterEngineWithExceptionMessage() {
394 OutputValidator validator = unpack("surefire-1857-exception-message", "-" + jupiter)
395 .setTestToRun("ExceptionMessageTest")
396 .sysProp("junit5.version", jupiter)
397 .maven()
398 .withFailure()
399 .executeTest()
400 .verifyTextInLog("ExceptionMessageTest.errorTest")
401 .assertTestSuiteResults(1, 1, 0, 0);
402
403 validator
404 .getSurefireReportsFile("TEST-jira1857.ExceptionMessageTest.xml", UTF_8)
405 .assertContainsText("message=\"error_message\"");
406 }
407
408 @Test
409 public void testJupiterEngineWithDisplayNames() throws VerificationException {
410 OutputValidator validator = unpack("junit-platform-engine-jupiter", "-" + jupiter)
411 .sysProp("junit5.version", jupiter)
412 .executeTest()
413 .verifyErrorFree(7);
414
415 validator
416 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest.txt", UTF_8)
417 .assertContainsText("<< ✨ >>");
418
419 validator
420 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest.txt", UTF_8)
421 .assertContainsText("Test set: << ✨ >>");
422
423 validator
424 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest.txt", UTF_8)
425 .assertContainsText(" -- in << ✨ >>");
426
427 validator
428 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest-output.txt", UTF_8)
429 .assertContainsText("<< ✨ >>");
430
431 validator
432 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest-output.txt", UTF_8)
433 .assertContainsText("73$71 ✔");
434
435 validator
436 .getSurefireReportsFile("junitplatformenginejupiter.DisplayNameTest-output.txt", UTF_8)
437 .assertContainsText("73$72 ✔");
438
439 validator
440 .getSurefireReportsFile("TEST-junitplatformenginejupiter.DisplayNameTest.xml", UTF_8)
441 .assertContainsText("testcase name=\"73$71 ✔\" classname=\"<< ✨ >>\"")
442 .assertContainsText("testcase name=\"73$72 ✔\" classname=\"<< ✨ >>\"")
443 .assertContainsText(XML_TESTSUITE_FRAGMENT);
444
445 validator
446 .getSurefireReportsFile("TEST-junitplatformenginejupiter.BasicJupiterTest.xml", UTF_8)
447 .assertContainsText("<testcase name=\"test(TestInfo)\" "
448 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"")
449 .assertContainsText("<testcase name=\"add(int, int, int) 0 + 1 = 1\" "
450 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"")
451 .assertContainsText("<testcase name=\"add(int, int, int) 1 + 2 = 3\" "
452 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"")
453 .assertContainsText("<testcase name=\"add(int, int, int) 49 + 51 = 100\" "
454 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"")
455 .assertContainsText("<testcase name=\"add(int, int, int) 1 + 100 = 101\" "
456 + "classname=\"junitplatformenginejupiter.BasicJupiterTest\"");
457
458 validator
459 .assertThatLogLine(matchesRegex(".*Running junitplatformenginejupiter.BasicJupiterTest$"), is(1))
460 .assertThatLogLine(matchesRegex(".*Tests run.* junitplatformenginejupiter.BasicJupiterTest$"), is(1))
461 .assertThatLogLine(matchesRegex(".*Running << . >>$"), is(1))
462 .assertThatLogLine(matchesRegex(".*Tests run.* << . >>$"), is(1));
463 }
464
465 @Test
466 public void testTags() {
467
468
469 assumeThat(jupiter, is(not("5.0.3")));
470
471 unpack("junit-platform-tags", "-" + jupiter)
472 .sysProp("junit5.version", jupiter)
473 .executeTest()
474 .verifyErrorFree(2);
475 }
476
477 private static String toRegex(String text) {
478 return text.replaceAll("\\.", "\\\\.")
479 .replaceAll("\\[", "\\\\[")
480 .replaceAll("]", "\\\\]")
481 .replaceAll("\\(", "\\\\(")
482 .replaceAll("\\)", "\\\\)")
483 .replaceAll("\\*", ".*");
484 }
485 }