[SUREFIRE-1057] Surefire doesn't detect TestNG test method which fail…

…s because of exception in @dataProvider method

surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1057DataProviderExceptionIT.java

@@ -0,0 +1,52 @@
+package org.apache.maven.surefire.its.jiras;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Test surefire-report on TestNG test
+ *
+ * @author <a href="mailto:[email protected]">Michal Bocek</a>
+ */
+public class Surefire1057DataProviderExceptionIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+
+    static final String NL = System.getProperty( "line.separator" );
+
+    @Test
+    public void testNgReport()
+        throws Exception
+    {
+        final OutputValidator outputValidator = unpack( "/surefire-1057-dataprovider-exception" )
+                        .addSurefireReportGoal()
+                        .executeCurrentGoals()
+                        .assertTestSuiteResults( 2, 0, 1, 1 );
+        outputValidator.getSurefireReportsXmlFile( "TEST-testng.DataProviderExceptionReportTest.xml" )
+                        .assertContainsText( "<skipped message=\"Skip tests message\" type=\"org.testng.SkipException\">org.testng.SkipException: Skip tests message");
+
+        outputValidator.getSurefireReportsXmlFile( "TEST-testng.DataProviderExceptionReportTest.xml" )
+                        .assertContainsText( "<failure message=\"&#10;Data Provider public java.lang.Object[] testng.DataProviderExceptionReportTest.dataProvider() must "
+                                           + "return either Object[][] or Iterator&lt;Object&gt;[], not class [Ljava.lang.Object;\" type=\"org.testng.TestNGException\"><![CDATA[org.testng.TestNGException:" );
+    }
+}

surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/pom.xml

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.maven.plugins.surefire</groupId>
+  <artifactId>surefire-1057-dataprovider-exception</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <name>Surefire 1057</name>
+
+  <profiles>
+    <profile>
+      <id>testng-old</id>
+      <activation>
+        <property><name>testNgClassifier</name></property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.testng</groupId>
+          <artifactId>testng</artifactId>
+          <version>${testNgVersion}</version>
+          <classifier>${testNgClassifier}</classifier>
+        </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <id>testng-new</id>
+      <activation>
+        <property><name>!testNgClassifier</name></property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.testng</groupId>
+          <artifactId>testng</artifactId>
+          <version>${testNgVersion}</version>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
+  <properties>
+     <testNgVersion>5.7</testNgVersion>
+     <testNgClassifier>jdk15</testNgClassifier>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>${surefire.version}</version>
+        <configuration>
+          <test>DataProviderExceptionReportTest</test>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.5</source>
+          <target>1.5</target>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/src/test/java/testng/DataProviderExceptionReportTest.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package testng;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+import org.testng.SkipException;
+
+public class DataProviderExceptionReportTest {
+
+	@Test(dataProvider = "provider")
+	public void testDataProvider() {
+	}
+
+	@DataProvider(name = "provider")
+	public Object[] dataProvider() {
+		throw new RuntimeException();
+	}
+
+	@Test
+	public void testSkip() {
+		throw new SkipException("Skip tests message");
+	}
+}

surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java

@@ -112,8 +112,12 @@ private static String getUserFriendlyTestName( ITestResult result )
 
     public void onTestSkipped( ITestResult result )
     {
-        ReportEntry report = new SimpleReportEntry( getSource( result ), getUserFriendlyTestName( result ) );
-
+        ReportEntry report = SimpleReportEntry.withException( getSource( result ), getUserFriendlyTestName( result ),
+                                                              new PojoStackTraceWriter(
+                                                                  result.getTestClass().getRealClass().getName(),
+                                                                  result.getMethod().getMethodName(),
+                                                                  result.getThrowable() ) );
+
         reporter.testSkipped( report );
     }