View Javadoc
1   package org.apache.maven.plugin.failsafe;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.io.IOException;
24  import java.util.Arrays;
25  import java.util.Collections;
26  import java.util.List;
27  
28  import org.apache.maven.artifact.Artifact;
29  import org.apache.maven.plugin.MojoExecutionException;
30  import org.apache.maven.plugin.MojoFailureException;
31  import org.apache.maven.plugin.surefire.AbstractSurefireMojo;
32  import org.apache.maven.plugin.surefire.booterclient.ChecksumCalculator;
33  import org.apache.maven.plugins.annotations.LifecyclePhase;
34  import org.apache.maven.plugins.annotations.Mojo;
35  import org.apache.maven.plugins.annotations.Parameter;
36  import org.apache.maven.plugins.annotations.ResolutionScope;
37  import org.apache.maven.shared.utils.ReaderFactory;
38  import org.apache.maven.shared.utils.StringUtils;
39  import org.apache.maven.surefire.suite.RunResult;
40  
41  /**
42   * Run integration tests using Surefire.
43   *
44   * @author Jason van Zyl
45   * @author Stephen Connolly
46   * @noinspection JavaDoc,
47   */
48  @Mojo( name = "integration-test", requiresProject = true, requiresDependencyResolution = ResolutionScope.TEST,
49         defaultPhase = LifecyclePhase.INTEGRATION_TEST, threadSafe = true )
50  public class IntegrationTestMojo
51      extends AbstractSurefireMojo
52  {
53  
54      private static final String FAILSAFE_IN_PROGRESS_CONTEXT_KEY = "failsafe-in-progress";
55  
56      /**
57       * The path representing project <em>JAR</em> file, if exists; Otherwise the directory containing generated
58       * classes of the project being tested. This will be included after the test classes in the test classpath.
59       */
60      @Parameter( defaultValue = "${project.build.outputDirectory}" )
61      private File classesDirectory;
62  
63      /**
64       * Set this to "true" to skip running integration tests, but still compile them. Its use is NOT RECOMMENDED, but
65       * quite convenient on occasion.
66       *
67       * @since 2.4.3-alpha-2
68       */
69      @Parameter( property = "skipITs" )
70      private boolean skipITs;
71  
72      /**
73       * Base directory where all reports are written to.
74       */
75      @Parameter( defaultValue = "${project.build.directory}/failsafe-reports" )
76      private File reportsDirectory;
77  
78      /**
79       * Specify this parameter to run individual tests by file name, overriding the <code>includes/excludes</code>
80       * parameters. Each pattern you specify here will be used to create an include pattern formatted like
81       * <code>**&#47;${test}.java</code>, so you can just type "-Dit.test=MyTest" to run a single test called
82       * "foo/MyTest.java".<br/>
83       * This parameter overrides the <code>includes/excludes</code> parameters, and the TestNG <code>suiteXmlFiles</code>
84       * parameter.
85       * <p/>
86       * Since 2.7.3 You can execute a limited number of methods in the test with adding #myMethod or #my*ethod. E.g. type
87       * "-Dit.test=MyTest#myMethod" <b>supported for junit 4.x and testNg</b>
88       * <br/>
89       * Since 2.19 a complex syntax is supported in one parameter (JUnit 4, JUnit 4.7+, TestNG):<br/>
90       * "-Dit.test=???IT, !Unstable*, pkg&#47;**&#47;Ci*leIT.java, *IT#test*One+testTwo?????, #fast*+slowTest"<br/>
91       * "-Dit.test=Basic*, !%regex[.*.Unstable.*], !%regex[.*.MyIT.class#one.*|two.*], %regex[#fast.*|slow.*]"<br/>
92       * <br/>
93       * The Parameterized JUnit runner <em>describes</em> test methods using an index in brackets, so the non-regex
94       * method pattern would become: <em>#testMethod[*]</em>. If using <em>@Parameters(name="{index}: fib({0})={1}")</em>
95       * and selecting the index e.g. 5 in pattern, the non-regex method pattern would become <em>#testMethod[5:*]</em>.
96       * <br/>
97       */
98      @Parameter( property = "it.test" )
99      private String test;
100 
101     /**
102      * The summary file to write integration test results to.
103      */
104     @Parameter( defaultValue = "${project.build.directory}/failsafe-reports/failsafe-summary.xml", required = true )
105     private File summaryFile;
106 
107     /**
108      * Option to print summary of test suites or just print the test cases that have errors.
109      */
110     @Parameter( property = "failsafe.printSummary", defaultValue = "true" )
111     private boolean printSummary;
112 
113     /**
114      * Selects the formatting for the test report to be generated. Can be set as "brief" or "plain".
115      * Only applies to the output format of the output files  (target/surefire-reports/testName.txt)
116      */
117     @Parameter( property = "failsafe.reportFormat", defaultValue = "brief" )
118     private String reportFormat;
119 
120     /**
121      * Option to generate a file test report or just output the test report to the console.
122      */
123     @Parameter( property = "failsafe.useFile", defaultValue = "true" )
124     private boolean useFile;
125 
126     /**
127      * Set this to "true" to cause a failure if the none of the tests specified in -Dtest=... are run. Defaults to
128      * "true".
129      *
130      * @since 2.12
131      */
132     @Parameter( property = "it.failIfNoSpecifiedTests" )
133     private Boolean failIfNoSpecifiedTests;
134 
135     /**
136      * Attach a debugger to the forked JVM. If set to "true", the process will suspend and wait for a debugger to attach
137      * on port 5005. If set to some other string, that string will be appended to the argLine, allowing you to configure
138      * arbitrary debuggability options (without overwriting the other options specified through the <code>argLine</code>
139      * parameter).
140      *
141      * @since 2.4
142      */
143     @Parameter( property = "maven.failsafe.debug" )
144     private String debugForkedProcess;
145 
146     /**
147      * Kill the forked test process after a certain number of seconds. If set to 0, wait forever for the process, never
148      * timing out.
149      *
150      * @since 2.4
151      */
152     @Parameter( property = "failsafe.timeout" )
153     private int forkedProcessTimeoutInSeconds;
154 
155     /**
156      * Stop executing queued parallel JUnit tests after a certain number of seconds.
157      * <br/>
158      * Example values: "3.5", "4"<br/>
159      * <br/>
160      * If set to 0, wait forever, never timing out.
161      * Makes sense with specified <code>parallel</code> different from "none".
162      *
163      * @since 2.16
164      */
165     @Parameter( property = "failsafe.parallel.timeout" )
166     private double parallelTestsTimeoutInSeconds;
167 
168     /**
169      * Stop executing queued parallel JUnit tests
170      * and <em>interrupt</em> currently running tests after a certain number of seconds.
171      * <br/>
172      * Example values: "3.5", "4"<br/>
173      * <br/>
174      * If set to 0, wait forever, never timing out.
175      * Makes sense with specified <code>parallel</code> different from "none".
176      *
177      * @since 2.16
178      */
179     @Parameter( property = "failsafe.parallel.forcedTimeout" )
180     private double parallelTestsTimeoutForcedInSeconds;
181 
182     /**
183      * A list of &lt;include> elements specifying the tests (by pattern) that should be included in testing. When not
184      * specified and when the <code>test</code> parameter is not specified, the default includes will be <code><br/>
185      * &lt;includes><br/>
186      * &nbsp;&lt;include>**&#47;IT*.java&lt;/include><br/>
187      * &nbsp;&lt;include>**&#47;*IT.java&lt;/include><br/>
188      * &nbsp;&lt;include>**&#47;*ITCase.java&lt;/include><br/>
189      * &lt;/includes><br/>
190      * </code>
191      * <p/>
192      * Each include item may also contain a comma-separated sublist of items, which will be treated as multiple
193      * &nbsp;&lt;include> entries.<br/>
194      * Since 2.19 a complex syntax is supported in one parameter (JUnit 4, JUnit 4.7+, TestNG):<br/>
195      * &nbsp;&lt;include>%regex[.*[Cat|Dog].*], Basic????, !Unstable*&lt;/include><br/>
196      * &nbsp;&lt;include>%regex[.*[Cat|Dog].*], !%regex[pkg.*Slow.*.class], pkg&#47;**&#47;*Fast*.java&lt;/include><br/>
197      * <p/>
198      * This parameter is ignored if the TestNG <code>suiteXmlFiles</code> parameter is specified.<br/>
199      * <br/>
200      * <em>Notice that</em> these values are relative to the directory containing generated test classes of the project
201      * being tested. This directory is declared by the parameter <code>testClassesDirectory</code> which defaults
202      * to the POM property <code>${project.build.testOutputDirectory}</code>, typically <em>src/test/java</em>
203      * unless overridden.
204      */
205     @Parameter
206     private List<String> includes;
207 
208     /**
209      * Option to pass dependencies to the system's classloader instead of using an isolated class loader when forking.
210      * Prevents problems with JDKs which implement the service provider lookup mechanism by using the system's
211      * classloader.
212      *
213      * @since 2.3
214      */
215     @Parameter( property = "failsafe.useSystemClassLoader", defaultValue = "true" )
216     private boolean useSystemClassLoader;
217 
218     /**
219      * By default, Surefire forks your tests using a manifest-only JAR; set this parameter to "false" to force it to
220      * launch your tests with a plain old Java classpath. (See the
221      * <a href="http://maven.apache.org/plugins/maven-failsafe-plugin/examples/class-loading.html">
222      *     http://maven.apache.org/plugins/maven-failsafe-plugin/examples/class-loading.html</a>;
223      * for a more detailed explanation of manifest-only JARs and their benefits.)
224      * <br/>
225      * Beware, setting this to "false" may cause your tests to fail on Windows if your classpath is too long.
226      *
227      * @since 2.4.3
228      */
229     @Parameter( property = "failsafe.useManifestOnlyJar", defaultValue = "true" )
230     private boolean useManifestOnlyJar;
231 
232     /**
233      * The character encoding scheme to be applied.
234      */
235     @Parameter( property = "encoding", defaultValue = "${project.reporting.outputEncoding}" )
236     private String encoding;
237 
238     /**
239      * (JUnit 4+ providers)
240      * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
241      * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
242      * However, all the failing attempts will be recorded.
243      */
244     @Parameter( property = "failsafe.rerunFailingTestsCount", defaultValue = "0" )
245     private int rerunFailingTestsCount;
246 
247     /**
248      * (TestNG) List of &lt;suiteXmlFile> elements specifying TestNG suite xml file locations. Note that
249      * <code>suiteXmlFiles</code> is incompatible with several other parameters of this plugin, like
250      * <code>includes/excludes</code>.<br/>
251      * This parameter is ignored if the <code>test</code> parameter is specified (allowing you to run a single test
252      * instead of an entire suite).
253      *
254      * @since 2.2
255      */
256     @Parameter( property = "failsafe.suiteXmlFiles" )
257     private File[] suiteXmlFiles;
258 
259     /**
260      * Defines the order the tests will be run in. Supported values are "alphabetical", "reversealphabetical", "random",
261      * "hourly" (alphabetical on even hours, reverse alphabetical on odd hours), "failedfirst", "balanced" and
262      * "filesystem".
263      * <br/>
264      * <br/>
265      * Odd/Even for hourly is determined at the time the of scanning the classpath, meaning it could change during a
266      * multi-module build.
267      * <br/>
268      * <br/>
269      * Failed first will run tests that failed on previous run first, as well as new tests for this run.
270      * <br/>
271      * <br/>
272      * Balanced is only relevant with parallel=classes, and will try to optimize the run-order of the tests reducing the
273      * overall execution time. Initially a statistics file is created and every next test run will reorder classes.
274      * <br/>
275      * <br/>
276      * Note that the statistics are stored in a file named .surefire-XXXXXXXXX beside pom.xml, and should not be checked
277      * into version control. The "XXXXX" is the SHA1 checksum of the entire surefire configuration, so different
278      * configurations will have different statistics files, meaning if you change any config settings you will re-run
279      * once before new statistics data can be established.
280      *
281      * @since 2.7
282      */
283     @Parameter( property = "failsafe.runOrder", defaultValue = "filesystem" )
284     private String runOrder;
285 
286     /**
287      * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are
288      * also specified, these patterns are appended. Example with path, simple and regex includes:<br/>
289      * &#042;&#047;test/*<br/>
290      * &#042;&#042;&#047;NotIncludedByDefault.java<br/>
291      * %regex[.*Test.*|.*Not.*]<br/>
292      */
293     @Parameter( property = "failsafe.includesFile" )
294     private File includesFile;
295 
296     /**
297      * A file containing exclude patterns. Blank lines, or lines starting with # are ignored. If {@code excludes} are
298      * also specified, these patterns are appended. Example with path, simple and regex excludes:<br/>
299      * &#042;&#047;test/*<br/>
300      * &#042;&#042;&#047;DontRunTest.*<br/>
301      * %regex[.*Test.*|.*Not.*]<br/>
302      */
303     @Parameter( property = "failsafe.excludesFile" )
304     private File excludesFile;
305 
306     /**
307      * Set to error/failure count in order to skip remaining tests.
308      * Due to race conditions in parallel/forked execution this may not be fully guaranteed.<br/>
309      * Enable with system property -Dfailsafe.skipAfterFailureCount=1 or any number greater than zero.
310      * Defaults to "0".<br/>
311      * See the prerequisites and limitations in documentation:<br/>
312      * <a href="http://maven.apache.org/plugins/maven-failsafe-plugin/examples/skip-after-failure.html">
313      *     http://maven.apache.org/plugins/maven-failsafe-plugin/examples/skip-after-failure.html</a>;
314      *
315      * @since 2.19
316      */
317     @Parameter( property = "failsafe.skipAfterFailureCount", defaultValue = "0" )
318     private int skipAfterFailureCount;
319 
320     /**
321      * After the plugin process is shutdown by sending SIGTERM signal (CTRL+C), SHUTDOWN command is received by every
322      * forked JVM. By default (shutdown=testset) forked JVM would not continue with new test which means that
323      * the current test may still continue to run.<br/>
324      * The parameter can be configured with other two values "exit" and "kill".<br/>
325      * Using "exit" forked JVM executes System.exit(1) after the plugin process has received SIGTERM signal.<br/>
326      * Using "kill" the JVM executes Runtime.halt(1) and kills itself.
327      *
328      * @since 2.19
329      */
330     @Parameter( property = "failsafe.shutdown", defaultValue = "testset" )
331     private String shutdown;
332 
333     protected int getRerunFailingTestsCount()
334     {
335         return rerunFailingTestsCount;
336     }
337 
338     @SuppressWarnings( "unchecked" )
339     protected void handleSummary( RunResult summary, Exception firstForkException )
340         throws MojoExecutionException, MojoFailureException
341     {
342         File summaryFile = getSummaryFile();
343         if ( !summaryFile.getParentFile().isDirectory() )
344         {
345             //noinspection ResultOfMethodCallIgnored
346             summaryFile.getParentFile().mkdirs();
347         }
348 
349         try
350         {
351             Object token = getPluginContext().get( FAILSAFE_IN_PROGRESS_CONTEXT_KEY );
352             summary.writeSummary( summaryFile, token != null, getEncodingOrDefault() );
353         }
354         catch ( IOException e )
355         {
356             throw new MojoExecutionException( e.getMessage(), e );
357         }
358 
359         getPluginContext().put( FAILSAFE_IN_PROGRESS_CONTEXT_KEY, FAILSAFE_IN_PROGRESS_CONTEXT_KEY );
360     }
361 
362     private String getEncodingOrDefault()
363     {
364         if ( StringUtils.isEmpty( encoding ) )
365         {
366             getLog().warn( "File encoding has not been set, using platform encoding " + ReaderFactory.FILE_ENCODING
367                            + ", i.e. build is platform dependent! The file encoding for reports output files "
368                                + "should be provided by the POM property ${project.reporting.outputEncoding}." );
369             return ReaderFactory.FILE_ENCODING;
370         }
371         else
372         {
373             return encoding;
374         }
375     }
376 
377     @SuppressWarnings( "deprecation" )
378     protected boolean isSkipExecution()
379     {
380         return isSkip() || isSkipTests() || isSkipITs() || isSkipExec();
381     }
382 
383     protected String getPluginName()
384     {
385         return "failsafe";
386     }
387 
388     protected String[] getDefaultIncludes()
389     {
390         return new String[]{ "**/IT*.java", "**/*IT.java", "**/*ITCase.java" };
391     }
392 
393     public boolean isSkipTests()
394     {
395         return skipTests;
396     }
397 
398     public void setSkipTests( boolean skipTests )
399     {
400         this.skipTests = skipTests;
401     }
402 
403     public boolean isSkipITs()
404     {
405         return skipITs;
406     }
407 
408     public void setSkipITs( boolean skipITs )
409     {
410         this.skipITs = skipITs;
411     }
412 
413     @SuppressWarnings( "deprecation" )
414     @Deprecated
415     public boolean isSkipExec()
416     {
417         return skipExec;
418     }
419 
420     @SuppressWarnings( "deprecation" )
421     @Deprecated
422     public void setSkipExec( boolean skipExec )
423     {
424         this.skipExec = skipExec;
425     }
426 
427     public boolean isSkip()
428     {
429         return skip;
430     }
431 
432     public void setSkip( boolean skip )
433     {
434         this.skip = skip;
435     }
436 
437     public File getBasedir()
438     {
439         return basedir;
440     }
441 
442     public void setBasedir( File basedir )
443     {
444         this.basedir = basedir;
445     }
446 
447     public File getTestClassesDirectory()
448     {
449         return testClassesDirectory;
450     }
451 
452     public void setTestClassesDirectory( File testClassesDirectory )
453     {
454         this.testClassesDirectory = testClassesDirectory;
455     }
456 
457     /**
458      * @return Output directory, or artifact file if artifact type is "jar". If not forking the JVM, parameter
459      * {@link #useSystemClassLoader} is ignored and the {@link org.apache.maven.surefire.booter.IsolatedClassLoader} is
460      * used instead. See the resolution of {@link #getClassLoaderConfiguration() ClassLoaderConfiguration}.
461      */
462     public File getClassesDirectory()
463     {
464         Artifact artifact = getProject().getArtifact();
465         File artifactFile = artifact.getFile();
466 
467         boolean useArtifactFile = artifactFile != null && artifactFile.isFile()
468             && artifactFile.getName().toLowerCase().endsWith( ".jar" );
469 
470         return useArtifactFile ? artifactFile : classesDirectory;
471     }
472 
473     public void setClassesDirectory( File classesDirectory )
474     {
475         this.classesDirectory = classesDirectory;
476     }
477 
478     public File getReportsDirectory()
479     {
480         return reportsDirectory;
481     }
482 
483     public void setReportsDirectory( File reportsDirectory )
484     {
485         this.reportsDirectory = reportsDirectory;
486     }
487 
488     public String getTest()
489     {
490         return test;
491     }
492 
493     public void setTest( String test )
494     {
495         this.test = test;
496     }
497 
498     public File getSummaryFile()
499     {
500         return summaryFile;
501     }
502 
503     public void setSummaryFile( File summaryFile )
504     {
505         this.summaryFile = summaryFile;
506     }
507 
508     public boolean isPrintSummary()
509     {
510         return printSummary;
511     }
512 
513     public void setPrintSummary( boolean printSummary )
514     {
515         this.printSummary = printSummary;
516     }
517 
518     public String getReportFormat()
519     {
520         return reportFormat;
521     }
522 
523     public void setReportFormat( String reportFormat )
524     {
525         this.reportFormat = reportFormat;
526     }
527 
528     public boolean isUseFile()
529     {
530         return useFile;
531     }
532 
533     public void setUseFile( boolean useFile )
534     {
535         this.useFile = useFile;
536     }
537 
538     public String getDebugForkedProcess()
539     {
540         return debugForkedProcess;
541     }
542 
543     public void setDebugForkedProcess( String debugForkedProcess )
544     {
545         this.debugForkedProcess = debugForkedProcess;
546     }
547 
548     public int getForkedProcessTimeoutInSeconds()
549     {
550         return forkedProcessTimeoutInSeconds;
551     }
552 
553     public void setForkedProcessTimeoutInSeconds( int forkedProcessTimeoutInSeconds )
554     {
555         this.forkedProcessTimeoutInSeconds = forkedProcessTimeoutInSeconds;
556     }
557 
558     public double getParallelTestsTimeoutInSeconds()
559     {
560         return parallelTestsTimeoutInSeconds;
561     }
562 
563     public void setParallelTestsTimeoutInSeconds( double parallelTestsTimeoutInSeconds )
564     {
565         this.parallelTestsTimeoutInSeconds = parallelTestsTimeoutInSeconds;
566     }
567 
568     public double getParallelTestsTimeoutForcedInSeconds()
569     {
570         return parallelTestsTimeoutForcedInSeconds;
571     }
572 
573     public void setParallelTestsTimeoutForcedInSeconds( double parallelTestsTimeoutForcedInSeconds )
574     {
575         this.parallelTestsTimeoutForcedInSeconds = parallelTestsTimeoutForcedInSeconds;
576     }
577 
578     public boolean isUseSystemClassLoader()
579     {
580         return useSystemClassLoader;
581     }
582 
583     public void setUseSystemClassLoader( boolean useSystemClassLoader )
584     {
585         this.useSystemClassLoader = useSystemClassLoader;
586     }
587 
588     public boolean isUseManifestOnlyJar()
589     {
590         return useManifestOnlyJar;
591     }
592 
593     public void setUseManifestOnlyJar( boolean useManifestOnlyJar )
594     {
595         this.useManifestOnlyJar = useManifestOnlyJar;
596     }
597 
598     // the following will be refactored out once the common code is all in one place
599 
600     public boolean isTestFailureIgnore()
601     {
602         return true; // ignore
603     }
604 
605     public void setTestFailureIgnore( boolean testFailureIgnore )
606     {
607         // ignore
608     }
609 
610     protected void addPluginSpecificChecksumItems( ChecksumCalculator checksum )
611     {
612         checksum.add( skipITs );
613         checksum.add( summaryFile );
614     }
615 
616     public Boolean getFailIfNoSpecifiedTests()
617     {
618         return failIfNoSpecifiedTests;
619     }
620 
621     public void setFailIfNoSpecifiedTests( boolean failIfNoSpecifiedTests )
622     {
623         this.failIfNoSpecifiedTests = failIfNoSpecifiedTests;
624     }
625 
626     public int getSkipAfterFailureCount()
627     {
628         return skipAfterFailureCount;
629     }
630 
631     public String getShutdown()
632     {
633         return shutdown;
634     }
635 
636     @Override
637     public List<String> getIncludes()
638     {
639         return includes;
640     }
641 
642     @Override
643     public void setIncludes( List<String> includes )
644     {
645         this.includes = includes;
646     }
647 
648     public File[] getSuiteXmlFiles()
649     {
650         return suiteXmlFiles.clone();
651     }
652 
653     @SuppressWarnings( "UnusedDeclaration" )
654     public void setSuiteXmlFiles( File[] suiteXmlFiles )
655     {
656         this.suiteXmlFiles = suiteXmlFiles.clone();
657     }
658 
659     public String getRunOrder()
660     {
661         return runOrder;
662     }
663 
664     @SuppressWarnings( "UnusedDeclaration" )
665     public void setRunOrder( String runOrder )
666     {
667         this.runOrder = runOrder;
668     }
669 
670     @Override
671     public File getIncludesFile()
672     {
673         return includesFile;
674     }
675 
676     @Override
677     public File getExcludesFile()
678     {
679         return excludesFile;
680     }
681 
682     @Override
683     protected final List<File> suiteXmlFiles()
684     {
685         return hasSuiteXmlFiles() ? Arrays.asList( suiteXmlFiles ) : Collections.<File>emptyList();
686     }
687 
688     @Override
689     protected final boolean hasSuiteXmlFiles()
690     {
691         return suiteXmlFiles != null && suiteXmlFiles.length != 0;
692     }
693 }