View Javadoc
1   package org.apache.maven.plugin.surefire;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.util.List;
24  import org.apache.maven.plugin.MojoExecutionException;
25  import org.apache.maven.plugin.MojoFailureException;
26  import org.apache.maven.plugins.annotations.LifecyclePhase;
27  import org.apache.maven.plugins.annotations.Mojo;
28  import org.apache.maven.plugins.annotations.Parameter;
29  import org.apache.maven.plugins.annotations.ResolutionScope;
30  import org.apache.maven.surefire.suite.RunResult;
31  
32  /**
33   * Run tests using Surefire.
34   *
35   * @author Jason van Zyl
36   * @noinspection JavaDoc
37   */
38  @Mojo( name = "test", defaultPhase = LifecyclePhase.TEST, threadSafe = true,
39         requiresDependencyResolution = ResolutionScope.TEST )
40  public class SurefirePlugin
41      extends AbstractSurefireMojo
42      implements SurefireReportParameters
43  {
44  
45      /**
46       * The directory containing generated classes of the project being tested. This will be included after the test
47       * classes in the test classpath.
48       */
49      @Parameter( defaultValue = "${project.build.outputDirectory}" )
50      private File classesDirectory;
51  
52      /**
53       * Set this to "true" to ignore a failure during testing. Its use is NOT RECOMMENDED, but quite convenient on
54       * occasion.
55       */
56      @Parameter( property = "maven.test.failure.ignore", defaultValue = "false" )
57      private boolean testFailureIgnore;
58  
59      /**
60       * Base directory where all reports are written to.
61       */
62      @Parameter( defaultValue = "${project.build.directory}/surefire-reports" )
63      private File reportsDirectory;
64  
65      /**
66       * Specify this parameter to run individual tests by file name, overriding the <code>includes/excludes</code>
67       * parameters. Each pattern you specify here will be used to create an include pattern formatted like
68       * <code>**&#47;${test}.java</code>, so you can just type "-Dtest=MyTest" to run a single test called
69       * "foo/MyTest.java". The test patterns prefixed with a <code>!</code> will be excluded.<br/>
70       * This parameter overrides the <code>includes/excludes</code> parameters, and the TestNG <code>suiteXmlFiles</code>
71       * parameter.
72       * <p/>
73       * Since 2.7.3, you can execute a limited number of methods in the test by adding #myMethod or #my*ethod. For
74       * example, "-Dtest=MyTest#myMethod". This is supported for junit 4.x and testNg.<br/>
75       * <br/>
76       * Since 2.19 a complex syntax is supported in one parameter (JUnit 4, JUnit 4.7+, TestNG):<br/>
77       * "-Dtest=???Test, !Unstable*, pkg&#47;**&#47;Ci*leTest.java, *Test#test*One+testTwo?????, #fast*+slowTest"<br/>
78       * "-Dtest=Basic*, !%regex[.*.Unstable.*], !%regex[.*.MyTest.class#one.*|two.*], %regex[#fast.*|slow.*]"<br/>
79       * <br/>
80       * The Parameterized JUnit runner <em>describes</em> test methods using an index in brackets, so the non-regex
81       * method pattern would become: <em>#testMethod[*]</em>. If using <em>@Parameters(name="{index}: fib({0})={1}")</em>
82       * and selecting the index e.g. 5 in pattern, the non-regex method pattern would become <em>#testMethod[5:*]</em>.
83       * <br/>
84       */
85      @Parameter( property = "test" )
86      private String test;
87  
88      /**
89       * Option to print summary of test suites or just print the test cases that have errors.
90       */
91      @Parameter( property = "surefire.printSummary", defaultValue = "true" )
92      private boolean printSummary;
93  
94      /**
95       * Selects the formatting for the test report to be generated. Can be set as "brief" or "plain".
96       * Only applies to the output format of the output files  (target/surefire-reports/testName.txt)
97       */
98      @Parameter( property = "surefire.reportFormat", defaultValue = "brief" )
99      private String reportFormat;
100 
101     /**
102      * Option to generate a file test report or just output the test report to the console.
103      */
104     @Parameter( property = "surefire.useFile", defaultValue = "true" )
105     private boolean useFile;
106 
107     /**
108      * Set this to "true" to cause a failure if the none of the tests specified in -Dtest=... are run. Defaults to
109      * "true".
110      *
111      * @since 2.12
112      */
113     @Parameter( property = "surefire.failIfNoSpecifiedTests" )
114     private Boolean failIfNoSpecifiedTests;
115 
116     /**
117      * Attach a debugger to the forked JVM. If set to "true", the process will suspend and wait for a debugger to attach
118      * on port 5005. If set to some other string, that string will be appended to the argLine, allowing you to configure
119      * arbitrary debuggability options (without overwriting the other options specified through the <code>argLine</code>
120      * parameter).
121      *
122      * @since 2.4
123      */
124     @Parameter( property = "maven.surefire.debug" )
125     private String debugForkedProcess;
126 
127     /**
128      * Kill the forked test process after a certain number of seconds. If set to 0, wait forever for the process, never
129      * timing out.
130      *
131      * @since 2.4
132      */
133     @Parameter( property = "surefire.timeout" )
134     private int forkedProcessTimeoutInSeconds;
135 
136     /**
137      * Stop executing queued parallel JUnit tests after a certain number of seconds.
138      * <br/>
139      * Example values: "3.5", "4"<br/>
140      * <br/>
141      * If set to 0, wait forever, never timing out.
142      * Makes sense with specified <code>parallel</code> different from "none".
143      *
144      * @since 2.16
145      */
146     @Parameter( property = "surefire.parallel.timeout" )
147     private double parallelTestsTimeoutInSeconds;
148 
149     /**
150      * Stop executing queued parallel JUnit tests
151      * and <em>interrupt</em> currently running tests after a certain number of seconds.
152      * <br/>
153      * Example values: "3.5", "4"<br/>
154      * <br/>
155      * If set to 0, wait forever, never timing out.
156      * Makes sense with specified <code>parallel</code> different from "none".
157      *
158      * @since 2.16
159      */
160     @Parameter( property = "surefire.parallel.forcedTimeout" )
161     private double parallelTestsTimeoutForcedInSeconds;
162 
163     /**
164      * A list of &lt;include> elements specifying the tests (by pattern) that should be included in testing. When not
165      * specified and when the <code>test</code> parameter is not specified, the default includes will be <code><br/>
166      * &lt;includes><br/>
167      * &nbsp;&lt;include>**&#47;Test*.java&lt;/include><br/>
168      * &nbsp;&lt;include>**&#47;*Test.java&lt;/include><br/>
169      * &nbsp;&lt;include>**&#47;*TestCase.java&lt;/include><br/>
170      * &lt;/includes><br/>
171      * </code>
172      * <p/>
173      * Each include item may also contain a comma-separated sublist of items, which will be treated as multiple
174      * &nbsp;&lt;include> entries.<br/>
175      * Since 2.19 a complex syntax is supported in one parameter (JUnit 4, JUnit 4.7+, TestNG):<br/>
176      * &nbsp;&lt;include>%regex[.*[Cat|Dog].*], Basic????, !Unstable*&lt;/include><br/>
177      * &nbsp;&lt;include>%regex[.*[Cat|Dog].*], !%regex[pkg.*Slow.*.class], pkg&#47;**&#47;*Fast*.java&lt;/include><br/>
178      * <p/>
179      * This parameter is ignored if the TestNG <code>suiteXmlFiles</code> parameter is specified.<br/>
180      * <br/>
181      * <em>Notice that</em> these values are relative to the directory containing generated test classes of the project
182      * being tested. This directory is declared by the parameter <code>testClassesDirectory</code> which defaults
183      * to the POM property <code>${project.build.testOutputDirectory}</code>, typically <em>src/test/java</em>
184      * unless overridden.
185      */
186     @Parameter
187     private List<String> includes;
188 
189     /**
190      * Option to pass dependencies to the system's classloader instead of using an isolated class loader when forking.
191      * Prevents problems with JDKs which implement the service provider lookup mechanism by using the system's
192      * classloader.
193      *
194      * @since 2.3
195      */
196     @Parameter( property = "surefire.useSystemClassLoader", defaultValue = "true" )
197     private boolean useSystemClassLoader;
198 
199     /**
200      * By default, Surefire forks your tests using a manifest-only JAR; set this parameter to "false" to force it to
201      * launch your tests with a plain old Java classpath. (See the
202      * <a href="http://maven.apache.org/plugins/maven-surefire-plugin/examples/class-loading.html">
203      *     http://maven.apache.org/plugins/maven-surefire-plugin/examples/class-loading.html</a>;
204      * for a more detailed explanation of manifest-only JARs and their benefits.)
205      * <br/>
206      * Beware, setting this to "false" may cause your tests to fail on Windows if your classpath is too long.
207      *
208      * @since 2.4.3
209      */
210     @Parameter( property = "surefire.useManifestOnlyJar", defaultValue = "true" )
211     private boolean useManifestOnlyJar;
212 
213     /**
214      * (JUnit 4+ providers)
215      * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
216      * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
217      * However, all the failing attempts will be recorded.
218      */
219     @Parameter( property = "surefire.rerunFailingTestsCount", defaultValue = "0" )
220     protected int rerunFailingTestsCount;
221 
222     /**
223      * (TestNG) List of &lt;suiteXmlFile> elements specifying TestNG suite xml file locations. Note that
224      * <code>suiteXmlFiles</code> is incompatible with several other parameters of this plugin, like
225      * <code>includes/excludes</code>.<br/>
226      * This parameter is ignored if the <code>test</code> parameter is specified (allowing you to run a single test
227      * instead of an entire suite).
228      *
229      * @since 2.2
230      */
231     @Parameter( property = "surefire.suiteXmlFiles" )
232     private File[] suiteXmlFiles;
233 
234     /**
235      * Defines the order the tests will be run in. Supported values are "alphabetical", "reversealphabetical", "random",
236      * "hourly" (alphabetical on even hours, reverse alphabetical on odd hours), "failedfirst", "balanced" and
237      * "filesystem".
238      * <br/>
239      * <br/>
240      * Odd/Even for hourly is determined at the time the of scanning the classpath, meaning it could change during a
241      * multi-module build.
242      * <br/>
243      * <br/>
244      * Failed first will run tests that failed on previous run first, as well as new tests for this run.
245      * <br/>
246      * <br/>
247      * Balanced is only relevant with parallel=classes, and will try to optimize the run-order of the tests reducing the
248      * overall execution time. Initially a statistics file is created and every next test run will reorder classes.
249      * <br/>
250      * <br/>
251      * Note that the statistics are stored in a file named .surefire-XXXXXXXXX beside pom.xml, and should not be checked
252      * into version control. The "XXXXX" is the SHA1 checksum of the entire surefire configuration, so different
253      * configurations will have different statistics files, meaning if you change any config settings you will re-run
254      * once before new statistics data can be established.
255      *
256      * @since 2.7
257      */
258     @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" )
259     protected String runOrder;
260 
261     /**
262      * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are
263      * also specified, these patterns are appended. Example with path, simple and regex includes:<br/>
264      * &#042;&#047;test/*<br/>
265      * &#042;&#042;&#047;NotIncludedByDefault.java<br/>
266      * %regex[.*Test.*|.*Not.*]<br/>
267      */
268     @Parameter( property = "surefire.includesFile" )
269     private File includesFile;
270 
271     /**
272      * A file containing exclude patterns. Blank lines, or lines starting with # are ignored. If {@code excludes} are
273      * also specified, these patterns are appended. Example with path, simple and regex excludes:<br/>
274      * &#042;&#047;test/*<br/>
275      * &#042;&#042;&#047;DontRunTest.*<br/>
276      * %regex[.*Test.*|.*Not.*]<br/>
277      */
278     @Parameter( property = "surefire.excludesFile" )
279     private File excludesFile;
280 
281     /**
282      * Set to error/failure count in order to skip remaining tests.
283      * Due to race conditions in parallel/forked execution this may not be fully guaranteed.<br/>
284      * Enable with system property -Dsurefire.skipAfterFailureCount=1 or any number greater than zero.
285      * Defaults to "0".<br/>
286      * See the prerequisites and limitations in documentation:<br/>
287      * <a href="http://maven.apache.org/plugins/maven-surefire-plugin/examples/skip-after-failure.html">
288      *     http://maven.apache.org/plugins/maven-surefire-plugin/examples/skip-after-failure.html</a>;
289      *
290      * @since 2.19
291      */
292     @Parameter( property = "surefire.skipAfterFailureCount", defaultValue = "0" )
293     private int skipAfterFailureCount;
294 
295     /**
296      * After the plugin process is shutdown by sending SIGTERM signal (CTRL+C), SHUTDOWN command is received by every
297      * forked JVM. By default (shutdown=testset) forked JVM would not continue with new test which means that
298      * the current test may still continue to run.<br/>
299      * The parameter can be configured with other two values "exit" and "kill".<br/>
300      * Using "exit" forked JVM executes System.exit(1) after the plugin process has received SIGTERM signal.<br/>
301      * Using "kill" the JVM executes Runtime.halt(1) and kills itself.
302      *
303      * @since 2.19
304      */
305     @Parameter( property = "surefire.shutdown", defaultValue = "testset" )
306     private String shutdown;
307 
308     protected int getRerunFailingTestsCount()
309     {
310         return rerunFailingTestsCount;
311     }
312 
313     protected void handleSummary( RunResult summary, Exception firstForkException )
314         throws MojoExecutionException, MojoFailureException
315     {
316         assertNoException( firstForkException );
317 
318         SurefireHelper.reportExecution( this, summary, getLog() );
319     }
320 
321     private void assertNoException( Exception firstForkException )
322         throws MojoFailureException
323     {
324         if ( firstForkException != null )
325         {
326             throw new MojoFailureException( firstForkException.getMessage(), firstForkException );
327         }
328     }
329 
330     private void assertNoFailureOrTimeout( Exception summary )
331         throws MojoFailureException
332     {
333         if ( summary != null )
334         {
335             throw new MojoFailureException( "Failure or timeout" );
336         }
337     }
338 
339     protected boolean isSkipExecution()
340     {
341         return isSkip() || isSkipTests() || isSkipExec();
342     }
343 
344     protected String getPluginName()
345     {
346         return "surefire";
347     }
348 
349     protected String[] getDefaultIncludes()
350     {
351         return new String[]{ "**/Test*.java", "**/*Test.java", "**/*TestCase.java" };
352     }
353 
354     // now for the implementation of the field accessors
355 
356     public boolean isSkipTests()
357     {
358         return skipTests;
359     }
360 
361     public void setSkipTests( boolean skipTests )
362     {
363         this.skipTests = skipTests;
364     }
365 
366     /**
367      * @noinspection deprecation
368      */
369     public boolean isSkipExec()
370     {
371         return skipExec;
372     }
373 
374     /**
375      * @noinspection deprecation
376      */
377     public void setSkipExec( boolean skipExec )
378     {
379         this.skipExec = skipExec;
380     }
381 
382     public boolean isSkip()
383     {
384         return skip;
385     }
386 
387     public void setSkip( boolean skip )
388     {
389         this.skip = skip;
390     }
391 
392     public boolean isTestFailureIgnore()
393     {
394         return testFailureIgnore;
395     }
396 
397     public void setTestFailureIgnore( boolean testFailureIgnore )
398     {
399         this.testFailureIgnore = testFailureIgnore;
400     }
401 
402     public File getBasedir()
403     {
404         return basedir;
405     }
406 
407     public void setBasedir( File basedir )
408     {
409         this.basedir = basedir;
410     }
411 
412     public File getTestClassesDirectory()
413     {
414         return testClassesDirectory;
415     }
416 
417     public void setTestClassesDirectory( File testClassesDirectory )
418     {
419         this.testClassesDirectory = testClassesDirectory;
420     }
421 
422     public File getClassesDirectory()
423     {
424         return classesDirectory;
425     }
426 
427     public void setClassesDirectory( File classesDirectory )
428     {
429         this.classesDirectory = classesDirectory;
430     }
431 
432     public File getReportsDirectory()
433     {
434         return reportsDirectory;
435     }
436 
437     public void setReportsDirectory( File reportsDirectory )
438     {
439         this.reportsDirectory = reportsDirectory;
440     }
441 
442     public String getTest()
443     {
444         return test;
445     }
446 
447     public boolean isUseSystemClassLoader()
448     {
449         return useSystemClassLoader;
450     }
451 
452     public void setUseSystemClassLoader( boolean useSystemClassLoader )
453     {
454         this.useSystemClassLoader = useSystemClassLoader;
455     }
456 
457     public boolean isUseManifestOnlyJar()
458     {
459         return useManifestOnlyJar;
460     }
461 
462     public void setUseManifestOnlyJar( boolean useManifestOnlyJar )
463     {
464         this.useManifestOnlyJar = useManifestOnlyJar;
465     }
466 
467     public Boolean getFailIfNoSpecifiedTests()
468     {
469         return failIfNoSpecifiedTests;
470     }
471 
472     public void setFailIfNoSpecifiedTests( boolean failIfNoSpecifiedTests )
473     {
474         this.failIfNoSpecifiedTests = failIfNoSpecifiedTests;
475     }
476 
477     public int getSkipAfterFailureCount()
478     {
479         return skipAfterFailureCount;
480     }
481 
482     public String getShutdown()
483     {
484         return shutdown;
485     }
486 
487     public boolean isPrintSummary()
488     {
489         return printSummary;
490     }
491 
492     public void setPrintSummary( boolean printSummary )
493     {
494         this.printSummary = printSummary;
495     }
496 
497     public String getReportFormat()
498     {
499         return reportFormat;
500     }
501 
502     public void setReportFormat( String reportFormat )
503     {
504         this.reportFormat = reportFormat;
505     }
506 
507     public boolean isUseFile()
508     {
509         return useFile;
510     }
511 
512     public void setUseFile( boolean useFile )
513     {
514         this.useFile = useFile;
515     }
516 
517     public String getDebugForkedProcess()
518     {
519         return debugForkedProcess;
520     }
521 
522     public void setDebugForkedProcess( String debugForkedProcess )
523     {
524         this.debugForkedProcess = debugForkedProcess;
525     }
526 
527     public int getForkedProcessTimeoutInSeconds()
528     {
529         return forkedProcessTimeoutInSeconds;
530     }
531 
532     public void setForkedProcessTimeoutInSeconds( int forkedProcessTimeoutInSeconds )
533     {
534         this.forkedProcessTimeoutInSeconds = forkedProcessTimeoutInSeconds;
535     }
536 
537     public double getParallelTestsTimeoutInSeconds()
538     {
539         return parallelTestsTimeoutInSeconds;
540     }
541 
542     public void setParallelTestsTimeoutInSeconds( double parallelTestsTimeoutInSeconds )
543     {
544         this.parallelTestsTimeoutInSeconds = parallelTestsTimeoutInSeconds;
545     }
546 
547     public double getParallelTestsTimeoutForcedInSeconds()
548     {
549         return parallelTestsTimeoutForcedInSeconds;
550     }
551 
552     public void setParallelTestsTimeoutForcedInSeconds( double parallelTestsTimeoutForcedInSeconds )
553     {
554         this.parallelTestsTimeoutForcedInSeconds = parallelTestsTimeoutForcedInSeconds;
555     }
556 
557     public void setTest( String test )
558     {
559         this.test = test;
560     }
561 
562     @Override
563     public List<String> getIncludes()
564     {
565         return includes;
566     }
567 
568     @Override
569     public void setIncludes( List<String> includes )
570     {
571         this.includes = includes;
572     }
573 
574     public File[] getSuiteXmlFiles()
575     {
576         return suiteXmlFiles;
577     }
578 
579     @SuppressWarnings( "UnusedDeclaration" )
580     public void setSuiteXmlFiles( File[] suiteXmlFiles )
581     {
582         this.suiteXmlFiles = suiteXmlFiles;
583     }
584 
585     public String getRunOrder()
586     {
587         return runOrder;
588     }
589 
590     @SuppressWarnings( "UnusedDeclaration" )
591     public void setRunOrder( String runOrder )
592     {
593         this.runOrder = runOrder;
594     }
595 
596     @Override
597     public File getIncludesFile()
598     {
599         return includesFile;
600     }
601 
602     @Override
603     public File getExcludesFile()
604     {
605         return excludesFile;
606     }
607 }