[SUREFIRE-1057] Surefire doesn't detect TestNG test method which fail…

…s because of exception in @dataProvider method

surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1057DataProviderExceptionIT.java

@@ -0,0 +1,81 @@
+package org.apache.maven.surefire.its.jiras;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.FileNotFoundException;
+
+import org.apache.maven.shared.utils.xml.Xpp3Dom;
+import org.apache.maven.shared.utils.xml.Xpp3DomBuilder;
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Test surefire-report on TestNG test
+ *
+ * @author <a href="mailto:[email protected]">Michal Bocek</a>
+ */
+public class Surefire1057DataProviderExceptionIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+
+    @Test
+    public void testNgReport()
+        throws Exception
+    {
+        final OutputValidator outputValidator = unpack( "/surefire-1057-dataprovider-exception" )
+                        .addSurefireReportGoal()
+                        .executeCurrentGoals()
+                        .assertTestSuiteResults( 1, 0, 0, 1 );
+        Xpp3Dom[] children = readTests( outputValidator, "testng.DataProviderExceptionReportTest" );
+        assertThat( "Report should contains only one test case", children.length, is( equalTo( 1 ) ));
+
+        Xpp3Dom test = children[0];
+        assertThat( "Not expected classname", test.getAttribute( "classname" ),
+                    is( equalTo( "testng.DataProviderExceptionReportTest" ) ) );
+        assertThat( "Not expected test name", test.getAttribute( "name" ),
+                    is( equalTo( "testDataProvider" ) ) );
+
+        children = test.getChildren( "skipped" );
+        assertThat( "Test should contains only one skipped element", children.length, is( equalTo( 1 ) ));
+
+        Xpp3Dom skipped = children[0];
+        assertThat( "Skipped element should contains message attribute", skipped.getAttribute( "message" ),
+                    is( equalTo( "java.lang.RuntimeException: Exception in data provider" ) ) );
+        assertThat( "Skipped element should contains type attribute", skipped.getAttribute( "type" ),
+                    is( equalTo( "java.lang.RuntimeException" ) ) );
+
+    }
+
+    private Xpp3Dom[] readTests( OutputValidator validator, String className )
+        throws FileNotFoundException
+    {
+        Xpp3Dom testResult =
+            Xpp3DomBuilder.build( validator.getSurefireReportsXmlFile( "TEST-" + className + ".xml" ).getFileInputStream(),
+                                  "UTF-8" );
+        Xpp3Dom[] children = testResult.getChildren( "testcase" );
+        return children;
+    }
+
+}

surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/pom.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.maven.plugins.surefire</groupId>
+  <artifactId>surefire-1057-dataprovider-exception</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <name>Surefire 1057</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.testng</groupId>
+      <artifactId>testng</artifactId>
+      <version>6.8.8</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>${surefire.version}</version>
+        <configuration>
+          <test>DataProviderExceptionReportTest</test>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.5</source>
+          <target>1.5</target>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/src/test/java/testng/DataProviderExceptionReportTest.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package testng;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+import org.testng.SkipException;
+
+public class DataProviderExceptionReportTest
+{
+
+    @DataProvider( name = "provider" )
+    public Object[][] dataProvider()
+    {
+        throw new RuntimeException("Exception in data provider");
+    }
+
+    @Test( dataProvider = "provider" )
+    public void testDataProvider()
+    {
+    }
+}

surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java

@@ -112,8 +112,12 @@ private static String getUserFriendlyTestName( ITestResult result )
 
     public void onTestSkipped( ITestResult result )
     {
-        ReportEntry report = new SimpleReportEntry( getSource( result ), getUserFriendlyTestName( result ) );
-
+        ReportEntry report = SimpleReportEntry.withException( getSource( result ), getUserFriendlyTestName( result ),
+                                                              new PojoStackTraceWriter(
+                                                                  result.getTestClass().getRealClass().getName(),
+                                                                  result.getMethod().getMethodName(),
+                                                                  result.getThrowable() ) );
+
         reporter.testSkipped( report );
     }