From d199d5aaf1500e1f11dfd6eeeadb520f77c80b06 Mon Sep 17 00:00:00 2001 From: Michal Bocek Date: Mon, 17 Nov 2014 21:33:47 +0100 Subject: [PATCH] [SUREFIRE-1057] Surefire doesn't detect TestNG test method which fails because of exception in @DataProvider method --- .../Surefire1057DataProviderExceptionIT.java | 81 +++++++++++++++++++ .../pom.xml | 60 ++++++++++++++ .../DataProviderExceptionReportTest.java | 38 +++++++++ .../maven/surefire/testng/TestNGReporter.java | 8 +- 4 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1057DataProviderExceptionIT.java create mode 100644 surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/pom.xml create mode 100644 surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/src/test/java/testng/DataProviderExceptionReportTest.java diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1057DataProviderExceptionIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1057DataProviderExceptionIT.java new file mode 100644 index 0000000000..b13bbc1066 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1057DataProviderExceptionIT.java @@ -0,0 +1,81 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import java.io.FileNotFoundException; + +import org.apache.maven.shared.utils.xml.Xpp3Dom; +import org.apache.maven.shared.utils.xml.Xpp3DomBuilder; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * Test surefire-report on TestNG test + * + * @author Michal Bocek + */ +public class Surefire1057DataProviderExceptionIT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void testNgReport() + throws Exception + { + final OutputValidator outputValidator = unpack( "/surefire-1057-dataprovider-exception" ) + .addSurefireReportGoal() + .executeCurrentGoals() + .assertTestSuiteResults( 1, 0, 0, 1 ); + Xpp3Dom[] children = readTests( outputValidator, "testng.DataProviderExceptionReportTest" ); + assertThat( "Report should contains only one test case", children.length, is( equalTo( 1 ) )); + + Xpp3Dom test = children[0]; + assertThat( "Not expected classname", test.getAttribute( "classname" ), + is( equalTo( "testng.DataProviderExceptionReportTest" ) ) ); + assertThat( "Not expected test name", test.getAttribute( "name" ), + is( equalTo( "testDataProvider" ) ) ); + + children = test.getChildren( "skipped" ); + assertThat( "Test should contains only one skipped element", children.length, is( equalTo( 1 ) )); + + Xpp3Dom skipped = children[0]; + assertThat( "Skipped element should contains message attribute", skipped.getAttribute( "message" ), + is( equalTo( "java.lang.RuntimeException: Exception in data provider" ) ) ); + assertThat( "Skipped element should contains type attribute", skipped.getAttribute( "type" ), + is( equalTo( "java.lang.RuntimeException" ) ) ); + + } + + private Xpp3Dom[] readTests( OutputValidator validator, String className ) + throws FileNotFoundException + { + Xpp3Dom testResult = + Xpp3DomBuilder.build( validator.getSurefireReportsXmlFile( "TEST-" + className + ".xml" ).getFileInputStream(), + "UTF-8" ); + Xpp3Dom[] children = testResult.getChildren( "testcase" ); + return children; + } + +} diff --git a/surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/pom.xml b/surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/pom.xml new file mode 100644 index 0000000000..add2cfb719 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/pom.xml @@ -0,0 +1,60 @@ + + + + + + 4.0.0 + + org.apache.maven.plugins.surefire + surefire-1057-dataprovider-exception + 1.0-SNAPSHOT + Surefire 1057 + + + + org.testng + testng + 6.8.8 + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + DataProviderExceptionReportTest + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.5 + 1.5 + + + + + \ No newline at end of file diff --git a/surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/src/test/java/testng/DataProviderExceptionReportTest.java b/surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/src/test/java/testng/DataProviderExceptionReportTest.java new file mode 100644 index 0000000000..1d3b7037cb --- /dev/null +++ b/surefire-integration-tests/src/test/resources/surefire-1057-dataprovider-exception/src/test/java/testng/DataProviderExceptionReportTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package testng; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import org.testng.SkipException; + +public class DataProviderExceptionReportTest +{ + + @DataProvider( name = "provider" ) + public Object[][] dataProvider() + { + throw new RuntimeException("Exception in data provider"); + } + + @Test( dataProvider = "provider" ) + public void testDataProvider() + { + } +} diff --git a/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java b/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java index e8ff18c388..e80f788e53 100644 --- a/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java +++ b/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java @@ -112,8 +112,12 @@ private static String getUserFriendlyTestName( ITestResult result ) public void onTestSkipped( ITestResult result ) { - ReportEntry report = new SimpleReportEntry( getSource( result ), getUserFriendlyTestName( result ) ); - + ReportEntry report = SimpleReportEntry.withException( getSource( result ), getUserFriendlyTestName( result ), + new PojoStackTraceWriter( + result.getTestClass().getRealClass().getName(), + result.getMethod().getMethodName(), + result.getThrowable() ) ); + reporter.testSkipped( report ); }