View Javadoc
1   package org.apache.maven.plugin.surefire;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.util.Arrays;
24  import java.util.Collections;
25  import java.util.List;
26  import org.apache.maven.plugin.MojoExecutionException;
27  import org.apache.maven.plugin.MojoFailureException;
28  import org.apache.maven.plugins.annotations.LifecyclePhase;
29  import org.apache.maven.plugins.annotations.Mojo;
30  import org.apache.maven.plugins.annotations.Parameter;
31  import org.apache.maven.plugins.annotations.ResolutionScope;
32  import org.apache.maven.surefire.suite.RunResult;
33  
34  import static org.apache.maven.plugin.surefire.SurefireHelper.reportExecution;
35  
36  /**
37   * Run tests using Surefire.
38   *
39   * @author Jason van Zyl
40   * @noinspection JavaDoc
41   */
42  @Mojo( name = "test", defaultPhase = LifecyclePhase.TEST, threadSafe = true,
43         requiresDependencyResolution = ResolutionScope.TEST )
44  public class SurefirePlugin
45      extends AbstractSurefireMojo
46      implements SurefireReportParameters
47  {
48  
49      /**
50       * The directory containing generated classes of the project being tested. This will be included after the test
51       * classes in the test classpath.
52       */
53      @Parameter( defaultValue = "${project.build.outputDirectory}" )
54      private File classesDirectory;
55  
56      /**
57       * Set this to "true" to ignore a failure during testing. Its use is NOT RECOMMENDED, but quite convenient on
58       * occasion.
59       */
60      @Parameter( property = "maven.test.failure.ignore", defaultValue = "false" )
61      private boolean testFailureIgnore;
62  
63      /**
64       * Base directory where all reports are written to.
65       */
66      @Parameter( defaultValue = "${project.build.directory}/surefire-reports" )
67      private File reportsDirectory;
68  
69      /**
70       * Specify this parameter to run individual tests by file name, overriding the <code>includes/excludes</code>
71       * parameters. Each pattern you specify here will be used to create an include pattern formatted like
72       * <code>**&#47;${test}.java</code>, so you can just type "-Dtest=MyTest" to run a single test called
73       * "foo/MyTest.java". The test patterns prefixed with a <code>!</code> will be excluded.<br/>
74       * This parameter overrides the <code>includes/excludes</code> parameters, and the TestNG <code>suiteXmlFiles</code>
75       * parameter.
76       * <p/>
77       * Since 2.7.3, you can execute a limited number of methods in the test by adding #myMethod or #my*ethod. For
78       * example, "-Dtest=MyTest#myMethod". This is supported for junit 4.x and testNg.<br/>
79       * <br/>
80       * Since 2.19 a complex syntax is supported in one parameter (JUnit 4, JUnit 4.7+, TestNG):<br/>
81       * "-Dtest=???Test, !Unstable*, pkg&#47;**&#47;Ci*leTest.java, *Test#test*One+testTwo?????, #fast*+slowTest"<br/>
82       * "-Dtest=Basic*, !%regex[.*.Unstable.*], !%regex[.*.MyTest.class#one.*|two.*], %regex[#fast.*|slow.*]"<br/>
83       * <br/>
84       * The Parameterized JUnit runner <em>describes</em> test methods using an index in brackets, so the non-regex
85       * method pattern would become: <em>#testMethod[*]</em>. If using <em>@Parameters(name="{index}: fib({0})={1}")</em>
86       * and selecting the index e.g. 5 in pattern, the non-regex method pattern would become <em>#testMethod[5:*]</em>.
87       * <br/>
88       */
89      @Parameter( property = "test" )
90      private String test;
91  
92      /**
93       * Option to print summary of test suites or just print the test cases that have errors.
94       */
95      @Parameter( property = "surefire.printSummary", defaultValue = "true" )
96      private boolean printSummary;
97  
98      /**
99       * Selects the formatting for the test report to be generated. Can be set as "brief" or "plain".
100      * Only applies to the output format of the output files  (target/surefire-reports/testName.txt)
101      */
102     @Parameter( property = "surefire.reportFormat", defaultValue = "brief" )
103     private String reportFormat;
104 
105     /**
106      * Option to generate a file test report or just output the test report to the console.
107      */
108     @Parameter( property = "surefire.useFile", defaultValue = "true" )
109     private boolean useFile;
110 
111     /**
112      * Set this to "true" to cause a failure if the none of the tests specified in -Dtest=... are run. Defaults to
113      * "true".
114      *
115      * @since 2.12
116      */
117     @Parameter( property = "surefire.failIfNoSpecifiedTests" )
118     private Boolean failIfNoSpecifiedTests;
119 
120     /**
121      * Attach a debugger to the forked JVM. If set to "true", the process will suspend and wait for a debugger to attach
122      * on port 5005. If set to some other string, that string will be appended to the argLine, allowing you to configure
123      * arbitrary debuggability options (without overwriting the other options specified through the <code>argLine</code>
124      * parameter).
125      *
126      * @since 2.4
127      */
128     @Parameter( property = "maven.surefire.debug" )
129     private String debugForkedProcess;
130 
131     /**
132      * Kill the forked test process after a certain number of seconds. If set to 0, wait forever for the process, never
133      * timing out.
134      *
135      * @since 2.4
136      */
137     @Parameter( property = "surefire.timeout" )
138     private int forkedProcessTimeoutInSeconds;
139 
140     /**
141      * Forked process is normally terminated without any significant delay after given tests have completed.
142      * If the particular tests started non-daemon Thread(s), the process hangs instead of been properly terminated
143      * by <em>System.exit()</em>. Use this parameter in order to determine the timeout of terminating the process.
144      * <a href="http://maven.apache.org/surefire/maven-surefire-plugin/examples/shutdown.html">see the documentation:
145      * http://maven.apache.org/surefire/maven-surefire-plugin/examples/shutdown.html</a>;
146      * Turns to default fallback value of 30 seconds if negative integer.
147      *
148      * @since 2.20
149      */
150     @Parameter( property = "surefire.exitTimeout", defaultValue = "30" )
151     private int forkedProcessExitTimeoutInSeconds;
152 
153     /**
154      * Stop executing queued parallel JUnit tests after a certain number of seconds.
155      * <br/>
156      * Example values: "3.5", "4"<br/>
157      * <br/>
158      * If set to 0, wait forever, never timing out.
159      * Makes sense with specified <code>parallel</code> different from "none".
160      *
161      * @since 2.16
162      */
163     @Parameter( property = "surefire.parallel.timeout" )
164     private double parallelTestsTimeoutInSeconds;
165 
166     /**
167      * Stop executing queued parallel JUnit tests
168      * and <em>interrupt</em> currently running tests after a certain number of seconds.
169      * <br/>
170      * Example values: "3.5", "4"<br/>
171      * <br/>
172      * If set to 0, wait forever, never timing out.
173      * Makes sense with specified <code>parallel</code> different from "none".
174      *
175      * @since 2.16
176      */
177     @Parameter( property = "surefire.parallel.forcedTimeout" )
178     private double parallelTestsTimeoutForcedInSeconds;
179 
180     /**
181      * A list of &lt;include> elements specifying the tests (by pattern) that should be included in testing. When not
182      * specified and when the <code>test</code> parameter is not specified, the default includes will be <code><br/>
183      * &lt;includes><br/>
184      * &nbsp;&lt;include>**&#47;Test*.java&lt;/include><br/>
185      * &nbsp;&lt;include>**&#47;*Test.java&lt;/include><br/>
186      * &nbsp;&lt;include>**&#47;*Tests.java&lt;/include><br/>
187      * &nbsp;&lt;include>**&#47;*TestCase.java&lt;/include><br/>
188      * &lt;/includes><br/>
189      * </code>
190      * <p/>
191      * Each include item may also contain a comma-separated sublist of items, which will be treated as multiple
192      * &nbsp;&lt;include> entries.<br/>
193      * Since 2.19 a complex syntax is supported in one parameter (JUnit 4, JUnit 4.7+, TestNG):<br/>
194      * &nbsp;&lt;include>%regex[.*[Cat|Dog].*], Basic????, !Unstable*&lt;/include><br/>
195      * &nbsp;&lt;include>%regex[.*[Cat|Dog].*], !%regex[pkg.*Slow.*.class], pkg&#47;**&#47;*Fast*.java&lt;/include><br/>
196      * <p/>
197      * This parameter is ignored if the TestNG <code>suiteXmlFiles</code> parameter is specified.<br/>
198      * <br/>
199      * <em>Notice that</em> these values are relative to the directory containing generated test classes of the project
200      * being tested. This directory is declared by the parameter <code>testClassesDirectory</code> which defaults
201      * to the POM property <code>${project.build.testOutputDirectory}</code>, typically <em>src/test/java</em>
202      * unless overridden.
203      */
204     @Parameter
205     private List<String> includes;
206 
207     /**
208      * Option to pass dependencies to the system's classloader instead of using an isolated class loader when forking.
209      * Prevents problems with JDKs which implement the service provider lookup mechanism by using the system's
210      * classloader.
211      *
212      * @since 2.3
213      */
214     @Parameter( property = "surefire.useSystemClassLoader", defaultValue = "true" )
215     private boolean useSystemClassLoader;
216 
217     /**
218      * By default, Surefire forks your tests using a manifest-only JAR; set this parameter to "false" to force it to
219      * launch your tests with a plain old Java classpath. (See the
220      * <a href="http://maven.apache.org/plugins/maven-surefire-plugin/examples/class-loading.html">
221      *     http://maven.apache.org/plugins/maven-surefire-plugin/examples/class-loading.html</a>;
222      * for a more detailed explanation of manifest-only JARs and their benefits.)
223      * <br/>
224      * Beware, setting this to "false" may cause your tests to fail on Windows if your classpath is too long.
225      *
226      * @since 2.4.3
227      */
228     @Parameter( property = "surefire.useManifestOnlyJar", defaultValue = "true" )
229     private boolean useManifestOnlyJar;
230 
231     /**
232      * (JUnit 4+ providers)
233      * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
234      * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
235      * However, all the failing attempts will be recorded.
236      */
237     @Parameter( property = "surefire.rerunFailingTestsCount", defaultValue = "0" )
238     private int rerunFailingTestsCount;
239 
240     /**
241      * (TestNG) List of &lt;suiteXmlFile> elements specifying TestNG suite xml file locations. Note that
242      * <code>suiteXmlFiles</code> is incompatible with several other parameters of this plugin, like
243      * <code>includes/excludes</code>.<br/>
244      * This parameter is ignored if the <code>test</code> parameter is specified (allowing you to run a single test
245      * instead of an entire suite).
246      *
247      * @since 2.2
248      */
249     @Parameter( property = "surefire.suiteXmlFiles" )
250     private File[] suiteXmlFiles;
251 
252     /**
253      * Defines the order the tests will be run in. Supported values are "alphabetical", "reversealphabetical", "random",
254      * "hourly" (alphabetical on even hours, reverse alphabetical on odd hours), "failedfirst", "balanced" and
255      * "filesystem".
256      * <br/>
257      * <br/>
258      * Odd/Even for hourly is determined at the time the of scanning the classpath, meaning it could change during a
259      * multi-module build.
260      * <br/>
261      * <br/>
262      * Failed first will run tests that failed on previous run first, as well as new tests for this run.
263      * <br/>
264      * <br/>
265      * Balanced is only relevant with parallel=classes, and will try to optimize the run-order of the tests reducing the
266      * overall execution time. Initially a statistics file is created and every next test run will reorder classes.
267      * <br/>
268      * <br/>
269      * Note that the statistics are stored in a file named .surefire-XXXXXXXXX beside pom.xml, and should not be checked
270      * into version control. The "XXXXX" is the SHA1 checksum of the entire surefire configuration, so different
271      * configurations will have different statistics files, meaning if you change any config settings you will re-run
272      * once before new statistics data can be established.
273      *
274      * @since 2.7
275      */
276     @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" )
277     private String runOrder;
278 
279     /**
280      * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are
281      * also specified, these patterns are appended. Example with path, simple and regex includes:<br/>
282      * &#042;&#047;test/*<br/>
283      * &#042;&#042;&#047;NotIncludedByDefault.java<br/>
284      * %regex[.*Test.*|.*Not.*]<br/>
285      */
286     @Parameter( property = "surefire.includesFile" )
287     private File includesFile;
288 
289     /**
290      * A file containing exclude patterns. Blank lines, or lines starting with # are ignored. If {@code excludes} are
291      * also specified, these patterns are appended. Example with path, simple and regex excludes:<br/>
292      * &#042;&#047;test/*<br/>
293      * &#042;&#042;&#047;DontRunTest.*<br/>
294      * %regex[.*Test.*|.*Not.*]<br/>
295      */
296     @Parameter( property = "surefire.excludesFile" )
297     private File excludesFile;
298 
299     /**
300      * Set to error/failure count in order to skip remaining tests.
301      * Due to race conditions in parallel/forked execution this may not be fully guaranteed.<br/>
302      * Enable with system property -Dsurefire.skipAfterFailureCount=1 or any number greater than zero.
303      * Defaults to "0".<br/>
304      * See the prerequisites and limitations in documentation:<br/>
305      * <a href="http://maven.apache.org/plugins/maven-surefire-plugin/examples/skip-after-failure.html">
306      *     http://maven.apache.org/plugins/maven-surefire-plugin/examples/skip-after-failure.html</a>;
307      *
308      * @since 2.19
309      */
310     @Parameter( property = "surefire.skipAfterFailureCount", defaultValue = "0" )
311     private int skipAfterFailureCount;
312 
313     /**
314      * After the plugin process is shutdown by sending SIGTERM signal (CTRL+C), SHUTDOWN command is received by every
315      * forked JVM. By default (shutdown=testset) forked JVM would not continue with new test which means that
316      * the current test may still continue to run.<br/>
317      * The parameter can be configured with other two values "exit" and "kill".<br/>
318      * Using "exit" forked JVM executes System.exit(1) after the plugin process has received SIGTERM signal.<br/>
319      * Using "kill" the JVM executes Runtime.halt(1) and kills itself.
320      *
321      * @since 2.19
322      */
323     @Parameter( property = "surefire.shutdown", defaultValue = "testset" )
324     private String shutdown;
325 
326     protected int getRerunFailingTestsCount()
327     {
328         return rerunFailingTestsCount;
329     }
330 
331     protected void handleSummary( RunResult summary, Exception firstForkException )
332         throws MojoExecutionException, MojoFailureException
333     {
334         reportExecution( this, summary, getConsoleLogger(), firstForkException );
335     }
336 
337     protected boolean isSkipExecution()
338     {
339         return isSkip() || isSkipTests() || isSkipExec();
340     }
341 
342     protected String getPluginName()
343     {
344         return "surefire";
345     }
346 
347     protected String[] getDefaultIncludes()
348     {
349         return new String[]{ "**/Test*.java", "**/*Test.java", "**/*Tests.java", "**/*TestCase.java" };
350     }
351 
352     @Override
353     protected String getReportSchemaLocation()
354     {
355         return "https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report.xsd";
356     }
357 
358     // now for the implementation of the field accessors
359 
360     public boolean isSkipTests()
361     {
362         return skipTests;
363     }
364 
365     public void setSkipTests( boolean skipTests )
366     {
367         this.skipTests = skipTests;
368     }
369 
370     /**
371      * @noinspection deprecation
372      */
373     public boolean isSkipExec()
374     {
375         return skipExec;
376     }
377 
378     /**
379      * @noinspection deprecation
380      */
381     public void setSkipExec( boolean skipExec )
382     {
383         this.skipExec = skipExec;
384     }
385 
386     public boolean isSkip()
387     {
388         return skip;
389     }
390 
391     public void setSkip( boolean skip )
392     {
393         this.skip = skip;
394     }
395 
396     public boolean isTestFailureIgnore()
397     {
398         return testFailureIgnore;
399     }
400 
401     public void setTestFailureIgnore( boolean testFailureIgnore )
402     {
403         this.testFailureIgnore = testFailureIgnore;
404     }
405 
406     public File getBasedir()
407     {
408         return basedir;
409     }
410 
411     public void setBasedir( File basedir )
412     {
413         this.basedir = basedir;
414     }
415 
416     public File getTestClassesDirectory()
417     {
418         return testClassesDirectory;
419     }
420 
421     public void setTestClassesDirectory( File testClassesDirectory )
422     {
423         this.testClassesDirectory = testClassesDirectory;
424     }
425 
426     public File getClassesDirectory()
427     {
428         return classesDirectory;
429     }
430 
431     public void setClassesDirectory( File classesDirectory )
432     {
433         this.classesDirectory = classesDirectory;
434     }
435 
436     public File getReportsDirectory()
437     {
438         return reportsDirectory;
439     }
440 
441     public void setReportsDirectory( File reportsDirectory )
442     {
443         this.reportsDirectory = reportsDirectory;
444     }
445 
446     public String getTest()
447     {
448         return test;
449     }
450 
451     public boolean isUseSystemClassLoader()
452     {
453         return useSystemClassLoader;
454     }
455 
456     public void setUseSystemClassLoader( boolean useSystemClassLoader )
457     {
458         this.useSystemClassLoader = useSystemClassLoader;
459     }
460 
461     public boolean isUseManifestOnlyJar()
462     {
463         return useManifestOnlyJar;
464     }
465 
466     public void setUseManifestOnlyJar( boolean useManifestOnlyJar )
467     {
468         this.useManifestOnlyJar = useManifestOnlyJar;
469     }
470 
471     public Boolean getFailIfNoSpecifiedTests()
472     {
473         return failIfNoSpecifiedTests;
474     }
475 
476     public void setFailIfNoSpecifiedTests( boolean failIfNoSpecifiedTests )
477     {
478         this.failIfNoSpecifiedTests = failIfNoSpecifiedTests;
479     }
480 
481     public int getSkipAfterFailureCount()
482     {
483         return skipAfterFailureCount;
484     }
485 
486     public String getShutdown()
487     {
488         return shutdown;
489     }
490 
491     public boolean isPrintSummary()
492     {
493         return printSummary;
494     }
495 
496     public void setPrintSummary( boolean printSummary )
497     {
498         this.printSummary = printSummary;
499     }
500 
501     public String getReportFormat()
502     {
503         return reportFormat;
504     }
505 
506     public void setReportFormat( String reportFormat )
507     {
508         this.reportFormat = reportFormat;
509     }
510 
511     public boolean isUseFile()
512     {
513         return useFile;
514     }
515 
516     public void setUseFile( boolean useFile )
517     {
518         this.useFile = useFile;
519     }
520 
521     public String getDebugForkedProcess()
522     {
523         return debugForkedProcess;
524     }
525 
526     public void setDebugForkedProcess( String debugForkedProcess )
527     {
528         this.debugForkedProcess = debugForkedProcess;
529     }
530 
531     public int getForkedProcessTimeoutInSeconds()
532     {
533         return forkedProcessTimeoutInSeconds;
534     }
535 
536     public void setForkedProcessTimeoutInSeconds( int forkedProcessTimeoutInSeconds )
537     {
538         this.forkedProcessTimeoutInSeconds = forkedProcessTimeoutInSeconds;
539     }
540 
541     public int getForkedProcessExitTimeoutInSeconds()
542     {
543         return forkedProcessExitTimeoutInSeconds;
544     }
545 
546     public void setForkedProcessExitTimeoutInSeconds( int forkedProcessExitTimeoutInSeconds )
547     {
548         this.forkedProcessExitTimeoutInSeconds = forkedProcessExitTimeoutInSeconds;
549     }
550 
551     public double getParallelTestsTimeoutInSeconds()
552     {
553         return parallelTestsTimeoutInSeconds;
554     }
555 
556     public void setParallelTestsTimeoutInSeconds( double parallelTestsTimeoutInSeconds )
557     {
558         this.parallelTestsTimeoutInSeconds = parallelTestsTimeoutInSeconds;
559     }
560 
561     public double getParallelTestsTimeoutForcedInSeconds()
562     {
563         return parallelTestsTimeoutForcedInSeconds;
564     }
565 
566     public void setParallelTestsTimeoutForcedInSeconds( double parallelTestsTimeoutForcedInSeconds )
567     {
568         this.parallelTestsTimeoutForcedInSeconds = parallelTestsTimeoutForcedInSeconds;
569     }
570 
571     public void setTest( String test )
572     {
573         this.test = test;
574     }
575 
576     @Override
577     public List<String> getIncludes()
578     {
579         return includes;
580     }
581 
582     @Override
583     public void setIncludes( List<String> includes )
584     {
585         this.includes = includes;
586     }
587 
588     public File[] getSuiteXmlFiles()
589     {
590         return suiteXmlFiles.clone();
591     }
592 
593     @SuppressWarnings( "UnusedDeclaration" )
594     public void setSuiteXmlFiles( File[] suiteXmlFiles )
595     {
596         this.suiteXmlFiles = suiteXmlFiles.clone();
597     }
598 
599     public String getRunOrder()
600     {
601         return runOrder;
602     }
603 
604     @SuppressWarnings( "UnusedDeclaration" )
605     public void setRunOrder( String runOrder )
606     {
607         this.runOrder = runOrder;
608     }
609 
610     @Override
611     public File getIncludesFile()
612     {
613         return includesFile;
614     }
615 
616     @Override
617     public File getExcludesFile()
618     {
619         return excludesFile;
620     }
621 
622     @Override
623     protected final List<File> suiteXmlFiles()
624     {
625         return hasSuiteXmlFiles() ? Arrays.asList( suiteXmlFiles ) : Collections.<File>emptyList();
626     }
627 
628     @Override
629     protected final boolean hasSuiteXmlFiles()
630     {
631         return suiteXmlFiles != null && suiteXmlFiles.length != 0;
632     }
633 }