View Javadoc
1   package org.apache.maven.plugin.failsafe;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.io.IOException;
24  import java.util.List;
25  
26  import org.apache.maven.artifact.Artifact;
27  import org.apache.maven.plugin.MojoExecutionException;
28  import org.apache.maven.plugin.MojoFailureException;
29  import org.apache.maven.plugin.surefire.AbstractSurefireMojo;
30  import org.apache.maven.plugin.surefire.booterclient.ChecksumCalculator;
31  import org.apache.maven.plugins.annotations.LifecyclePhase;
32  import org.apache.maven.plugins.annotations.Mojo;
33  import org.apache.maven.plugins.annotations.Parameter;
34  import org.apache.maven.plugins.annotations.ResolutionScope;
35  import org.apache.maven.shared.utils.ReaderFactory;
36  import org.apache.maven.shared.utils.StringUtils;
37  import org.apache.maven.surefire.suite.RunResult;
38  
39  /**
40   * Run integration tests using Surefire.
41   *
42   * @author Jason van Zyl
43   * @author Stephen Connolly
44   * @noinspection JavaDoc,
45   */
46  @Mojo( name = "integration-test", requiresProject = true, requiresDependencyResolution = ResolutionScope.TEST,
47         defaultPhase = LifecyclePhase.INTEGRATION_TEST, threadSafe = true )
48  public class IntegrationTestMojo
49      extends AbstractSurefireMojo
50  {
51  
52      private static final String FAILSAFE_IN_PROGRESS_CONTEXT_KEY = "failsafe-in-progress";
53  
54      /**
55       * The path representing project <em>JAR</em> file, if exists; Otherwise the directory containing generated
56       * classes of the project being tested. This will be included after the test classes in the test classpath.
57       */
58      @Parameter( defaultValue = "${project.build.outputDirectory}" )
59      private File classesDirectory;
60  
61      /**
62       * Set this to "true" to skip running integration tests, but still compile them. Its use is NOT RECOMMENDED, but
63       * quite convenient on occasion.
64       *
65       * @since 2.4.3-alpha-2
66       */
67      @Parameter( property = "skipITs" )
68      private boolean skipITs;
69  
70      /**
71       * Base directory where all reports are written to.
72       */
73      @Parameter( defaultValue = "${project.build.directory}/failsafe-reports" )
74      private File reportsDirectory;
75  
76      /**
77       * Specify this parameter to run individual tests by file name, overriding the <code>includes/excludes</code>
78       * parameters. Each pattern you specify here will be used to create an include pattern formatted like
79       * <code>**&#47;${test}.java</code>, so you can just type "-Dit.test=MyTest" to run a single test called
80       * "foo/MyTest.java".<br/>
81       * This parameter overrides the <code>includes/excludes</code> parameters, and the TestNG <code>suiteXmlFiles</code>
82       * parameter.
83       * <p/>
84       * Since 2.7.3 You can execute a limited number of methods in the test with adding #myMethod or #my*ethod. E.g. type
85       * "-Dit.test=MyTest#myMethod" <b>supported for junit 4.x and testNg</b>
86       * <br/>
87       * Since 2.19 a complex syntax is supported in one parameter (JUnit 4, JUnit 4.7+, TestNG):<br/>
88       * "-Dit.test=???IT, !Unstable*, pkg&#47;**&#47;Ci*leIT.java, *IT#test*One+testTwo?????, #fast*+slowTest"<br/>
89       * "-Dit.test=Basic*, !%regex[.*.Unstable.*], !%regex[.*.MyIT.class#one.*|two.*], %regex[#fast.*|slow.*]"<br/>
90       * <br/>
91       * The Parameterized JUnit runner <em>describes</em> test methods using an index in brackets, so the non-regex
92       * method pattern would become: <em>#testMethod[*]</em>. If using <em>@Parameters(name="{index}: fib({0})={1}")</em>
93       * and selecting the index e.g. 5 in pattern, the non-regex method pattern would become <em>#testMethod[5:*]</em>.
94       * <br/>
95       */
96      @Parameter( property = "it.test" )
97      private String test;
98  
99      /**
100      * The summary file to write integration test results to.
101      */
102     @Parameter( defaultValue = "${project.build.directory}/failsafe-reports/failsafe-summary.xml", required = true )
103     private File summaryFile;
104 
105     /**
106      * Option to print summary of test suites or just print the test cases that have errors.
107      */
108     @Parameter( property = "failsafe.printSummary", defaultValue = "true" )
109     private boolean printSummary;
110 
111     /**
112      * Selects the formatting for the test report to be generated. Can be set as "brief" or "plain".
113      * Only applies to the output format of the output files  (target/surefire-reports/testName.txt)
114      */
115     @Parameter( property = "failsafe.reportFormat", defaultValue = "brief" )
116     private String reportFormat;
117 
118     /**
119      * Option to generate a file test report or just output the test report to the console.
120      */
121     @Parameter( property = "failsafe.useFile", defaultValue = "true" )
122     private boolean useFile;
123 
124     /**
125      * Set this to "true" to cause a failure if the none of the tests specified in -Dtest=... are run. Defaults to
126      * "true".
127      *
128      * @since 2.12
129      */
130     @Parameter( property = "it.failIfNoSpecifiedTests" )
131     private Boolean failIfNoSpecifiedTests;
132 
133     /**
134      * Attach a debugger to the forked JVM. If set to "true", the process will suspend and wait for a debugger to attach
135      * on port 5005. If set to some other string, that string will be appended to the argLine, allowing you to configure
136      * arbitrary debuggability options (without overwriting the other options specified through the <code>argLine</code>
137      * parameter).
138      *
139      * @since 2.4
140      */
141     @Parameter( property = "maven.failsafe.debug" )
142     private String debugForkedProcess;
143 
144     /**
145      * Kill the forked test process after a certain number of seconds. If set to 0, wait forever for the process, never
146      * timing out.
147      *
148      * @since 2.4
149      */
150     @Parameter( property = "failsafe.timeout" )
151     private int forkedProcessTimeoutInSeconds;
152 
153     /**
154      * Stop executing queued parallel JUnit tests after a certain number of seconds.
155      * <br/>
156      * Example values: "3.5", "4"<br/>
157      * <br/>
158      * If set to 0, wait forever, never timing out.
159      * Makes sense with specified <code>parallel</code> different from "none".
160      *
161      * @since 2.16
162      */
163     @Parameter( property = "failsafe.parallel.timeout" )
164     private double parallelTestsTimeoutInSeconds;
165 
166     /**
167      * Stop executing queued parallel JUnit tests
168      * and <em>interrupt</em> currently running tests after a certain number of seconds.
169      * <br/>
170      * Example values: "3.5", "4"<br/>
171      * <br/>
172      * If set to 0, wait forever, never timing out.
173      * Makes sense with specified <code>parallel</code> different from "none".
174      *
175      * @since 2.16
176      */
177     @Parameter( property = "failsafe.parallel.forcedTimeout" )
178     private double parallelTestsTimeoutForcedInSeconds;
179 
180     /**
181      * A list of &lt;include> elements specifying the tests (by pattern) that should be included in testing. When not
182      * specified and when the <code>test</code> parameter is not specified, the default includes will be <code><br/>
183      * &lt;includes><br/>
184      * &nbsp;&lt;include>**&#47;IT*.java&lt;/include><br/>
185      * &nbsp;&lt;include>**&#47;*IT.java&lt;/include><br/>
186      * &nbsp;&lt;include>**&#47;*ITCase.java&lt;/include><br/>
187      * &lt;/includes><br/>
188      * </code>
189      * <p/>
190      * Each include item may also contain a comma-separated sublist of items, which will be treated as multiple
191      * &nbsp;&lt;include> entries.<br/>
192      * Since 2.19 a complex syntax is supported in one parameter (JUnit 4, JUnit 4.7+, TestNG):<br/>
193      * &nbsp;&lt;include>%regex[.*[Cat|Dog].*], Basic????, !Unstable*&lt;/include><br/>
194      * &nbsp;&lt;include>%regex[.*[Cat|Dog].*], !%regex[pkg.*Slow.*.class], pkg&#47;**&#47;*Fast*.java&lt;/include><br/>
195      * <p/>
196      * This parameter is ignored if the TestNG <code>suiteXmlFiles</code> parameter is specified.<br/>
197      * <br/>
198      * <em>Notice that</em> these values are relative to the directory containing generated test classes of the project
199      * being tested. This directory is declared by the parameter <code>testClassesDirectory</code> which defaults
200      * to the POM property <code>${project.build.testOutputDirectory}</code>, typically <em>src/test/java</em>
201      * unless overridden.
202      */
203     @Parameter
204     private List<String> includes;
205 
206     /**
207      * Option to pass dependencies to the system's classloader instead of using an isolated class loader when forking.
208      * Prevents problems with JDKs which implement the service provider lookup mechanism by using the system's
209      * classloader.
210      *
211      * @since 2.3
212      */
213     @Parameter( property = "failsafe.useSystemClassLoader", defaultValue = "true" )
214     private boolean useSystemClassLoader;
215 
216     /**
217      * By default, Surefire forks your tests using a manifest-only JAR; set this parameter to "false" to force it to
218      * launch your tests with a plain old Java classpath. (See the
219      * <a href="http://maven.apache.org/plugins/maven-failsafe-plugin/examples/class-loading.html">
220      *     http://maven.apache.org/plugins/maven-failsafe-plugin/examples/class-loading.html</a>;
221      * for a more detailed explanation of manifest-only JARs and their benefits.)
222      * <br/>
223      * Beware, setting this to "false" may cause your tests to fail on Windows if your classpath is too long.
224      *
225      * @since 2.4.3
226      */
227     @Parameter( property = "failsafe.useManifestOnlyJar", defaultValue = "true" )
228     private boolean useManifestOnlyJar;
229 
230     /**
231      * The character encoding scheme to be applied.
232      */
233     @Parameter( property = "encoding", defaultValue = "${project.reporting.outputEncoding}" )
234     private String encoding;
235 
236     /**
237      * (JUnit 4+ providers)
238      * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
239      * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
240      * However, all the failing attempts will be recorded.
241      */
242     @Parameter( property = "failsafe.rerunFailingTestsCount", defaultValue = "0" )
243     protected int rerunFailingTestsCount;
244 
245     /**
246      * (TestNG) List of &lt;suiteXmlFile> elements specifying TestNG suite xml file locations. Note that
247      * <code>suiteXmlFiles</code> is incompatible with several other parameters of this plugin, like
248      * <code>includes/excludes</code>.<br/>
249      * This parameter is ignored if the <code>test</code> parameter is specified (allowing you to run a single test
250      * instead of an entire suite).
251      *
252      * @since 2.2
253      */
254     @Parameter( property = "failsafe.suiteXmlFiles" )
255     private File[] suiteXmlFiles;
256 
257     /**
258      * Defines the order the tests will be run in. Supported values are "alphabetical", "reversealphabetical", "random",
259      * "hourly" (alphabetical on even hours, reverse alphabetical on odd hours), "failedfirst", "balanced" and
260      * "filesystem".
261      * <br/>
262      * <br/>
263      * Odd/Even for hourly is determined at the time the of scanning the classpath, meaning it could change during a
264      * multi-module build.
265      * <br/>
266      * <br/>
267      * Failed first will run tests that failed on previous run first, as well as new tests for this run.
268      * <br/>
269      * <br/>
270      * Balanced is only relevant with parallel=classes, and will try to optimize the run-order of the tests reducing the
271      * overall execution time. Initially a statistics file is created and every next test run will reorder classes.
272      * <br/>
273      * <br/>
274      * Note that the statistics are stored in a file named .surefire-XXXXXXXXX beside pom.xml, and should not be checked
275      * into version control. The "XXXXX" is the SHA1 checksum of the entire surefire configuration, so different
276      * configurations will have different statistics files, meaning if you change any config settings you will re-run
277      * once before new statistics data can be established.
278      *
279      * @since 2.7
280      */
281     @Parameter( property = "failsafe.runOrder", defaultValue = "filesystem" )
282     protected String runOrder;
283 
284     /**
285      * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are
286      * also specified, these patterns are appended. Example with path, simple and regex includes:<br/>
287      * &#042;&#047;test/*<br/>
288      * &#042;&#042;&#047;NotIncludedByDefault.java<br/>
289      * %regex[.*Test.*|.*Not.*]<br/>
290      */
291     @Parameter( property = "failsafe.includesFile" )
292     private File includesFile;
293 
294     /**
295      * A file containing exclude patterns. Blank lines, or lines starting with # are ignored. If {@code excludes} are
296      * also specified, these patterns are appended. Example with path, simple and regex excludes:<br/>
297      * &#042;&#047;test/*<br/>
298      * &#042;&#042;&#047;DontRunTest.*<br/>
299      * %regex[.*Test.*|.*Not.*]<br/>
300      */
301     @Parameter( property = "failsafe.excludesFile" )
302     private File excludesFile;
303 
304     /**
305      * Set to error/failure count in order to skip remaining tests.
306      * Due to race conditions in parallel/forked execution this may not be fully guaranteed.<br/>
307      * Enable with system property -Dfailsafe.skipAfterFailureCount=1 or any number greater than zero.
308      * Defaults to "0".<br/>
309      * See the prerequisites and limitations in documentation:<br/>
310      * <a href="http://maven.apache.org/plugins/maven-failsafe-plugin/examples/skip-after-failure.html">
311      *     http://maven.apache.org/plugins/maven-failsafe-plugin/examples/skip-after-failure.html</a>;
312      *
313      * @since 2.19
314      */
315     @Parameter( property = "failsafe.skipAfterFailureCount", defaultValue = "0" )
316     private int skipAfterFailureCount;
317 
318     /**
319      * After the plugin process is shutdown by sending SIGTERM signal (CTRL+C), SHUTDOWN command is received by every
320      * forked JVM. By default (shutdown=testset) forked JVM would not continue with new test which means that
321      * the current test may still continue to run.<br/>
322      * The parameter can be configured with other two values "exit" and "kill".<br/>
323      * Using "exit" forked JVM executes System.exit(1) after the plugin process has received SIGTERM signal.<br/>
324      * Using "kill" the JVM executes Runtime.halt(1) and kills itself.
325      *
326      * @since 2.19
327      */
328     @Parameter( property = "failsafe.shutdown", defaultValue = "testset" )
329     private String shutdown;
330 
331     protected int getRerunFailingTestsCount()
332     {
333         return rerunFailingTestsCount;
334     }
335 
336     @SuppressWarnings( "unchecked" )
337     protected void handleSummary( RunResult summary, Exception firstForkException )
338         throws MojoExecutionException, MojoFailureException
339     {
340         File summaryFile = getSummaryFile();
341         if ( !summaryFile.getParentFile().isDirectory() )
342         {
343             //noinspection ResultOfMethodCallIgnored
344             summaryFile.getParentFile().mkdirs();
345         }
346 
347         try
348         {
349             Object token = getPluginContext().get( FAILSAFE_IN_PROGRESS_CONTEXT_KEY );
350             summary.writeSummary( summaryFile, token != null, getEncodingOrDefault() );
351         }
352         catch ( IOException e )
353         {
354             throw new MojoExecutionException( e.getMessage(), e );
355         }
356 
357         getPluginContext().put( FAILSAFE_IN_PROGRESS_CONTEXT_KEY, FAILSAFE_IN_PROGRESS_CONTEXT_KEY );
358     }
359 
360     private String getEncodingOrDefault()
361     {
362         if ( StringUtils.isEmpty( encoding ) )
363         {
364             getLog().warn( "File encoding has not been set, using platform encoding " + ReaderFactory.FILE_ENCODING
365                            + ", i.e. build is platform dependent! The file encoding for reports output files "
366                                + "should be provided by the POM property ${project.reporting.outputEncoding}." );
367             return ReaderFactory.FILE_ENCODING;
368         }
369         else
370         {
371             return encoding;
372         }
373     }
374 
375     @SuppressWarnings( "deprecation" )
376     protected boolean isSkipExecution()
377     {
378         return isSkip() || isSkipTests() || isSkipITs() || isSkipExec();
379     }
380 
381     protected String getPluginName()
382     {
383         return "failsafe";
384     }
385 
386     protected String[] getDefaultIncludes()
387     {
388         return new String[]{ "**/IT*.java", "**/*IT.java", "**/*ITCase.java" };
389     }
390 
391     public boolean isSkipTests()
392     {
393         return skipTests;
394     }
395 
396     public void setSkipTests( boolean skipTests )
397     {
398         this.skipTests = skipTests;
399     }
400 
401     public boolean isSkipITs()
402     {
403         return skipITs;
404     }
405 
406     public void setSkipITs( boolean skipITs )
407     {
408         this.skipITs = skipITs;
409     }
410 
411     @SuppressWarnings( "deprecation" )
412     @Deprecated
413     public boolean isSkipExec()
414     {
415         return skipExec;
416     }
417 
418     @SuppressWarnings( "deprecation" )
419     @Deprecated
420     public void setSkipExec( boolean skipExec )
421     {
422         this.skipExec = skipExec;
423     }
424 
425     public boolean isSkip()
426     {
427         return skip;
428     }
429 
430     public void setSkip( boolean skip )
431     {
432         this.skip = skip;
433     }
434 
435     public File getBasedir()
436     {
437         return basedir;
438     }
439 
440     public void setBasedir( File basedir )
441     {
442         this.basedir = basedir;
443     }
444 
445     public File getTestClassesDirectory()
446     {
447         return testClassesDirectory;
448     }
449 
450     public void setTestClassesDirectory( File testClassesDirectory )
451     {
452         this.testClassesDirectory = testClassesDirectory;
453     }
454 
455     /**
456      * @return Output directory, or artifact file if artifact type is "jar". If not forking the JVM, parameter
457      * {@link #useSystemClassLoader} is ignored and the {@link org.apache.maven.surefire.booter.IsolatedClassLoader} is
458      * used instead. See the resolution of {@link #getClassLoaderConfiguration() ClassLoaderConfiguration}.
459      */
460     public File getClassesDirectory()
461     {
462         Artifact artifact = getProject().getArtifact();
463         File artifactFile = artifact.getFile();
464 
465         boolean useArtifactFile = artifactFile != null && artifactFile.isFile()
466             && artifactFile.getName().toLowerCase().endsWith( ".jar" );
467 
468         return useArtifactFile ? artifactFile : classesDirectory;
469     }
470 
471     public void setClassesDirectory( File classesDirectory )
472     {
473         this.classesDirectory = classesDirectory;
474     }
475 
476     public File getReportsDirectory()
477     {
478         return reportsDirectory;
479     }
480 
481     public void setReportsDirectory( File reportsDirectory )
482     {
483         this.reportsDirectory = reportsDirectory;
484     }
485 
486     public String getTest()
487     {
488         return test;
489     }
490 
491     public void setTest( String test )
492     {
493         this.test = test;
494     }
495 
496     public File getSummaryFile()
497     {
498         return summaryFile;
499     }
500 
501     public void setSummaryFile( File summaryFile )
502     {
503         this.summaryFile = summaryFile;
504     }
505 
506     public boolean isPrintSummary()
507     {
508         return printSummary;
509     }
510 
511     public void setPrintSummary( boolean printSummary )
512     {
513         this.printSummary = printSummary;
514     }
515 
516     public String getReportFormat()
517     {
518         return reportFormat;
519     }
520 
521     public void setReportFormat( String reportFormat )
522     {
523         this.reportFormat = reportFormat;
524     }
525 
526     public boolean isUseFile()
527     {
528         return useFile;
529     }
530 
531     public void setUseFile( boolean useFile )
532     {
533         this.useFile = useFile;
534     }
535 
536     public String getDebugForkedProcess()
537     {
538         return debugForkedProcess;
539     }
540 
541     public void setDebugForkedProcess( String debugForkedProcess )
542     {
543         this.debugForkedProcess = debugForkedProcess;
544     }
545 
546     public int getForkedProcessTimeoutInSeconds()
547     {
548         return forkedProcessTimeoutInSeconds;
549     }
550 
551     public void setForkedProcessTimeoutInSeconds( int forkedProcessTimeoutInSeconds )
552     {
553         this.forkedProcessTimeoutInSeconds = forkedProcessTimeoutInSeconds;
554     }
555 
556     public double getParallelTestsTimeoutInSeconds()
557     {
558         return parallelTestsTimeoutInSeconds;
559     }
560 
561     public void setParallelTestsTimeoutInSeconds( double parallelTestsTimeoutInSeconds )
562     {
563         this.parallelTestsTimeoutInSeconds = parallelTestsTimeoutInSeconds;
564     }
565 
566     public double getParallelTestsTimeoutForcedInSeconds()
567     {
568         return parallelTestsTimeoutForcedInSeconds;
569     }
570 
571     public void setParallelTestsTimeoutForcedInSeconds( double parallelTestsTimeoutForcedInSeconds )
572     {
573         this.parallelTestsTimeoutForcedInSeconds = parallelTestsTimeoutForcedInSeconds;
574     }
575 
576     public boolean isUseSystemClassLoader()
577     {
578         return useSystemClassLoader;
579     }
580 
581     public void setUseSystemClassLoader( boolean useSystemClassLoader )
582     {
583         this.useSystemClassLoader = useSystemClassLoader;
584     }
585 
586     public boolean isUseManifestOnlyJar()
587     {
588         return useManifestOnlyJar;
589     }
590 
591     public void setUseManifestOnlyJar( boolean useManifestOnlyJar )
592     {
593         this.useManifestOnlyJar = useManifestOnlyJar;
594     }
595 
596     // the following will be refactored out once the common code is all in one place
597 
598     public boolean isTestFailureIgnore()
599     {
600         return true; // ignore
601     }
602 
603     public void setTestFailureIgnore( boolean testFailureIgnore )
604     {
605         // ignore
606     }
607 
608     protected void addPluginSpecificChecksumItems( ChecksumCalculator checksum )
609     {
610         checksum.add( skipITs );
611         checksum.add( summaryFile );
612     }
613 
614     public Boolean getFailIfNoSpecifiedTests()
615     {
616         return failIfNoSpecifiedTests;
617     }
618 
619     public void setFailIfNoSpecifiedTests( boolean failIfNoSpecifiedTests )
620     {
621         this.failIfNoSpecifiedTests = failIfNoSpecifiedTests;
622     }
623 
624     public int getSkipAfterFailureCount()
625     {
626         return skipAfterFailureCount;
627     }
628 
629     public String getShutdown()
630     {
631         return shutdown;
632     }
633 
634     @Override
635     public List<String> getIncludes()
636     {
637         return includes;
638     }
639 
640     @Override
641     public void setIncludes( List<String> includes )
642     {
643         this.includes = includes;
644     }
645 
646     public File[] getSuiteXmlFiles()
647     {
648         return suiteXmlFiles;
649     }
650 
651     @SuppressWarnings( "UnusedDeclaration" )
652     public void setSuiteXmlFiles( File[] suiteXmlFiles )
653     {
654         this.suiteXmlFiles = suiteXmlFiles;
655     }
656 
657     public String getRunOrder()
658     {
659         return runOrder;
660     }
661 
662     @SuppressWarnings( "UnusedDeclaration" )
663     public void setRunOrder( String runOrder )
664     {
665         this.runOrder = runOrder;
666     }
667 
668     @Override
669     public File getIncludesFile()
670     {
671         return includesFile;
672     }
673 
674     @Override
675     public File getExcludesFile()
676     {
677         return excludesFile;
678     }
679 }