View Javadoc
1   /*
2    * JBoss, Home of Professional Open Source
3    * Copyright 2009, Red Hat Middleware LLC, and individual contributors
4    * by the @authors tag. See the copyright.txt in the distribution for a
5    * full listing of individual contributors.
6    *
7    * Licensed under the Apache License, Version 2.0 (the "License");
8    * you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   * http://www.apache.org/licenses/LICENSE-2.0
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.maven.surefire.junitcore;
18  
19  import junit.framework.Assert;
20  import junit.framework.TestCase;
21  import org.junit.BeforeClass;
22  import org.junit.Test;
23  import org.junit.runner.Result;
24  import org.junit.runner.notification.Failure;
25  
26  /**
27   * TestCase that expose "No tests were executed!" on Test failure using Maven Surefire 2.6-SNAPSHOT
28   * and the JUnit 4.8 Runner.
29   * {@code
30   * * <p/>
31   * -------------------------------------------------------
32   * T E S T S
33   * -------------------------------------------------------
34   * <p/>
35   * Results :
36   * <p/>
37   * Tests run: 0, Failures: 0, Errors: 0, Skipped: 0
38   * <p/>
39   * [INFO] ------------------------------------------------------------------------
40   * [INFO] BUILD FAILURE
41   * [INFO] ------------------------------------------------------------------------
42   * [INFO] Total time: 11.011s
43   * [INFO] Finished at: Thu Jul 15 13:59:14 CEST 2010
44   * [INFO] Final Memory: 24M/355M
45   * [INFO] ------------------------------------------------------------------------
46   * [ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:2.5:test
47   * (default-test) on project xxxxxx: No tests were executed!  (Set -DfailIfNoTests=false to
48   * ignore this error.) -> [Help 1]
49   * <p/>
50   * <p/>
51   * <dependency>
52   * <groupId>junit</groupId>
53   * <artifactId>junit</artifactId>
54   * <version>4.8.1</version>
55   * <scope>test</scope>
56   * </dependency>
57   * <p/>
58   * <dependency>
59   * <groupId>org.apache.maven.surefire</groupId>
60   * <artifactId>surefire-booter</artifactId>
61   * <version>2.6-SNAPSHOT</version>
62   * <scope>test</scope>
63   * </dependency>
64   * <dependency>
65   * <groupId>org.apache.maven.plugins</groupId>
66   * <artifactId>maven-surefire-plugin</artifactId>
67   * <version>2.6-SNAPSHOT</version>
68   * <scope>test</scope>
69   * </dependency>
70   * <dependency>
71   * <groupId>org.apache.maven.surefire</groupId>
72   * <artifactId>surefire-junit48</artifactId>
73   * <version>2.6-SNAPSHOT</version>
74   * <scope>test</scope>
75   * </dependency>
76   * }
77   *
78   * @author <a href="mailto:aslak@redhat.com">Aslak Knutsen</a>
79   * @version $Revision: $
80   */
81  public class MavenSurefireJUnit48RunnerTest
82      extends TestCase
83  {
84  
85      /*
86      * Assumption:
87      * The ConcurrentReportingRunListener assumes a Test will be Started before it Fails or Finishes.
88      *
89      * Reality:
90      * JUnits ParentRunner is responsible for adding the BeforeClass/AfterClass statements to the
91      * statement execution chain. After BeforeClass is executed, a Statement that delegates to the
92      * abstract method: runChild(T child, RunNotifier notifier) is called. As the JavaDoc explains:
93      * "Subclasses are responsible for making sure that relevant test events are reported through {@code notifier}".
94      * When a @BeforeClass fail, the child that should handle the relevant test events(Started, Failed, Finished)
95      * is never executed.
96      *
97      * Result:
98      * When Test Failed event is received in ConcurrentReportingRunListener without a Started event received first,
99      * it causes a NullPointException because there is no ClassReporter setup for that class yet. When this Exception
100     * is thrown from the ConcurrentReportingRunListener, JUnit catches the exception and reports is as a Failed test.
101     * But to avoid a wild loop, it removes the failing Listener before calling Failed test again. Since the
102     * ConcurrentReportingRunListener now is removed from the chain it will never receive the RunFinished event
103     * and the recorded state will never be replayed on the ReportManager.
104     *
105     * The End result: ReporterManager falsely believe no Test were run.
106     *
107     */
108     @SuppressWarnings( { "unchecked", "ThrowableResultOfMethodCallIgnored" } )
109     public void testSurefireShouldBeAbleToReportRunStatusEvenWithFailingTests()
110         throws Exception
111     {
112         Result result = new JUnitCoreTester().run( false, FailingTestClassTestNot.class );
113 
114         Assert.assertEquals( "JUnit should report correctly number of test ran(Finished)", 0, result.getRunCount() );
115 
116         for ( Failure failure : result.getFailures() )
117         {
118             System.out.println( failure.getException().getMessage() );
119         }
120 
121         Assert.assertEquals( "There should only be one Exception reported, the one from the failing TestCase", 1,
122                              result.getFailureCount() );
123 
124         Assert.assertEquals( "The exception thrown by the failing TestCase", RuntimeException.class,
125                              result.getFailures().get( 0 ).getException().getClass() );
126 
127     }
128 
129     /**
130      * Simple TestCase to force a Exception in @BeforeClass.
131      */
132     public static class FailingTestClassTestNot
133     {
134         @BeforeClass
135         public static void failingBeforeClass()
136             throws Exception
137         {
138             throw new RuntimeException( "Opps, we failed in @BeforeClass" );
139         }
140 
141         @Test
142         public void shouldNeverBeCalled()
143             throws Exception
144         {
145             Assert.assertTrue( true );
146         }
147     }
148 }