From 8d045a85252c56ac786b5065f6e74fd38ebcb1a7 Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 12:27:06 -0500 Subject: [PATCH 01/14] Feat: Add xsd validation to acceptance testing --- .../JUnitTestLoggerAcceptanceTests.cs | 8 + .../JunitXmlValidator.cs | 58 +++++ test/assets/JUnit.xsd | 212 ++++++++++++++++++ 3 files changed, 278 insertions(+) create mode 100644 test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs create mode 100644 test/assets/JUnit.xsd diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs index 253714f..93f0804 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs @@ -92,5 +92,13 @@ public void TestResultFileShouldContainTestCases() Assert.IsTrue(testcases.Where(x => x.Descendants().Any()) .All(x => x.Descendants().First().Attribute("type").Value == "failure")); } + + [TestMethod] + public void LoggedXmlValidatesAgainstXsdSchema() + { + var validator = new JunitXmlValidator(); + var result = validator.IsValid(File.ReadAllText(this.resultsFile)); + Assert.IsTrue(result); + } } } diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs new file mode 100644 index 0000000..0f55236 --- /dev/null +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs @@ -0,0 +1,58 @@ +// Copyright (c) Spekt Contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace JUnit.Xml.TestLogger.AcceptanceTests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using System.Xml; + using System.Xml.Linq; + using System.Xml.Schema; + + public class JunitXmlValidator + { + /// + /// Field is provided only to simplify debugging test failures. + /// + private readonly List failures = new List(); + + public bool IsValid(string xml) + { + var xmlReader = new StringReader(xml); + var xsdReader = new StringReader( + File.ReadAllText( + Path.Combine("..", "..", "..", "..", "assets", "JUnit.xsd"))); + + var schema = XmlSchema.Read( + xsdReader, + (sender, args) => { throw new XmlSchemaValidationException(args.Message, args.Exception); }); + + var xmlReaderSettings = new XmlReaderSettings(); + xmlReaderSettings.Schemas.Add(schema); + xmlReaderSettings.ValidationType = ValidationType.Schema; + + var veh = new ValidationEventHandler(this.BooksSettingsValidationEventHandler); + + xmlReaderSettings.ValidationEventHandler += veh; + using (XmlReader books = XmlReader.Create(xmlReader, xmlReaderSettings)) + { + while (books.Read()) + { + } + } + + xmlReaderSettings.ValidationEventHandler -= veh; + + return this.failures.Any() == false; + } + + private void BooksSettingsValidationEventHandler(object sender, ValidationEventArgs e) + { + this.failures.Add(e.Exception); + } + } +} diff --git a/test/assets/JUnit.xsd b/test/assets/JUnit.xsd new file mode 100644 index 0000000..320779e --- /dev/null +++ b/test/assets/JUnit.xsd @@ -0,0 +1,212 @@ + + + + JUnit test result schema for the Apache Ant JUnit and JUnitReport tasks +Copyright © 2011, Windy Road Technology Pty. Limited +The Apache Ant JUnit XML Schema is distributed under the terms of the Apache License Version 2.0 http://www.apache.org/licenses/ +Permission to waive conditions of this license may be requested from Windy Road Support (http://windyroad.org/support). + + + + + + + + + + Contains an aggregation of testsuite results + + + + + + + + + + Derived from testsuite/@name in the non-aggregated documents + + + + + Starts at '0' for the first testsuite and is incremented by 1 for each following testsuite + + + + + + + + + + + + Contains the results of exexuting a testsuite + + + + + Properties (e.g., environment settings) set during test execution + + + + + + + + + + + + + + + + + + + + + + + + + Indicates that the test errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. Contains as a text node relevant data for the error, e.g., a stack trace + + + + + + + The error message. e.g., if a java exception is thrown, the return value of getMessage() + + + + + The type of error that occured. e.g., if a java execption is thrown the full class name of the exception. + + + + + + + + + Indicates that the test failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals. Contains as a text node relevant data for the failure, e.g., a stack trace + + + + + + + The message specified in the assert + + + + + The type of the assert. + + + + + + + + + + Name of the test method + + + + + Full class name for the class the test method is in. + + + + + Time taken (in seconds) to execute the test + + + + + + + Data that was written to standard out while the test was executed + + + + + + + + + + Data that was written to standard error while the test was executed + + + + + + + + + + + Full class name of the test for non-aggregated testsuite documents. Class name without the package for aggregated testsuites documents + + + + + + + + + + when the test was executed. Timezone may not be specified. + + + + + Host on which the tests were executed. 'localhost' should be used if the hostname cannot be determined. + + + + + + + + + + The total number of tests in the suite + + + + + The total number of tests in the suite that failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals + + + + + The total number of tests in the suite that errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. + + + + + The total number of ignored or skipped tests in the suite. + + + + + Time taken (in seconds) to execute the tests in the suite + + + + + + + + + \ No newline at end of file From 0024424ed9d23aac9e5d22cf49f9e63b7e52f4ae Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 12:33:00 -0500 Subject: [PATCH 02/14] Fix: Resolve incorrect timestamp format for #17 --- src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index 3ca70c2..3b0db2f 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -40,8 +40,6 @@ public class JUnitXmlTestLogger : ITestLoggerWithParameters private const string ResultStatusPassed = "Passed"; private const string ResultStatusFailed = "Failed"; - private const string DateFormat = "yyyy-MM-ddT HH:mm:ssZ"; - // Tokens to allow user to manipulate output file or directory names. private const string AssemblyToken = "{assembly}"; private const string FrameworkToken = "{framework}"; @@ -50,7 +48,7 @@ public class JUnitXmlTestLogger : ITestLoggerWithParameters private string outputFilePath; private List results; - private DateTime localStartTime; + private DateTime utcStartTime; public enum MethodFormat { @@ -370,7 +368,7 @@ private void InitializeImpl(TestLoggerEvents events, string outputPath) this.results = new List(); } - this.localStartTime = DateTime.UtcNow; + this.utcStartTime = DateTime.UtcNow; } private XElement CreateTestSuitesElement(List results) @@ -403,7 +401,7 @@ private XElement CreateTestSuiteElement(List results) element.SetAttributeValue("failures", results.Where(x => x.Outcome == TestOutcome.Failed).Count()); element.SetAttributeValue("errors", 0); // looks like this isn't supported by .net? element.SetAttributeValue("time", results.Sum(x => x.Duration.TotalSeconds)); - element.SetAttributeValue("timestamp", this.localStartTime.ToString(DateFormat, CultureInfo.InvariantCulture)); + element.SetAttributeValue("timestamp", this.utcStartTime.ToString("s")); element.SetAttributeValue("hostname", results.First().TestCase.ExecutorUri); return element; From 6b06e36c95cb64de05a74cc40bcee5590adac37d Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 12:58:20 -0500 Subject: [PATCH 03/14] Fix: Resolve extra and missing elemnts. Fix unit test which required attributes not in the xsd --- src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs | 11 ++++------- .../JUnitTestLoggerAcceptanceTests.cs | 5 ----- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index 3b0db2f..5149a2a 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -379,12 +379,6 @@ private XElement CreateTestSuitesElement(List results) var element = new XElement("testsuites", testsuiteElements); - element.SetAttributeValue("name", Path.GetFileName(results.First().AssemblyPath)); - - element.SetAttributeValue("tests", results.Count); - element.SetAttributeValue("failures", results.Where(x => x.Outcome == TestOutcome.Failed).Count()); - element.SetAttributeValue("time", results.Sum(x => x.Duration.TotalSeconds)); - return element; } @@ -392,7 +386,8 @@ private XElement CreateTestSuiteElement(List results) { var testCaseElements = results.Select(a => this.CreateTestCaseElement(a)); - var element = new XElement("testsuite", testCaseElements); + // Adding required properties element without any contents, which must occur before test cases elements. + var element = new XElement("testsuite", new XElement("properties"), testCaseElements); element.SetAttributeValue("name", Path.GetFileName(results.First().AssemblyPath)); @@ -403,6 +398,8 @@ private XElement CreateTestSuiteElement(List results) element.SetAttributeValue("time", results.Sum(x => x.Duration.TotalSeconds)); element.SetAttributeValue("timestamp", this.utcStartTime.ToString("s")); element.SetAttributeValue("hostname", results.First().TestCase.ExecutorUri); + element.SetAttributeValue("id", 0); // we never output multiple, so this is always zero. + element.SetAttributeValue("package", Path.GetFileName(results.First().AssemblyPath)); return element; } diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs index 93f0804..94b55b5 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs @@ -46,11 +46,6 @@ public void TestResultFileShouldContainTestSuitesInformation() var node = this.resultsXml.XPathSelectElement("/testsuites"); Assert.IsNotNull(node); - Assert.AreEqual("JUnit.Xml.TestLogger.NetCore.Tests.dll", node.Attribute(XName.Get("name")).Value); - Assert.AreEqual("52", node.Attribute(XName.Get("tests")).Value); - Assert.AreEqual("14", node.Attribute(XName.Get("failures")).Value); - - Convert.ToDouble(node.Attribute(XName.Get("time")).Value); } [TestMethod] From d2df6ffc5a3b7caa174415a9c697dad54cf3b4d9 Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 13:06:29 -0500 Subject: [PATCH 04/14] Fix: Resolve xsd validation issue. From #16 --- src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index 5149a2a..4acbadb 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -426,7 +426,7 @@ private XElement CreateTestCaseElement(TestResultInfo result) } testcaseElement.SetAttributeValue("file", result.TestCase.Source); - testcaseElement.SetAttributeValue("time", result.Duration.TotalSeconds); + testcaseElement.SetAttributeValue("time", result.Duration.TotalSeconds.ToString("0.000000")); if (result.Outcome == TestOutcome.Failed) { From be3ebc7ae74f8380683367ec62ec5e57bcadf3b4 Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 13:13:44 -0500 Subject: [PATCH 05/14] Fix Remove attribute not specified in xsd. --- src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index 4acbadb..be842d1 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -425,7 +425,6 @@ private XElement CreateTestCaseElement(TestResultInfo result) testcaseElement.SetAttributeValue("name", result.Name); } - testcaseElement.SetAttributeValue("file", result.TestCase.Source); testcaseElement.SetAttributeValue("time", result.Duration.TotalSeconds.ToString("0.000000")); if (result.Outcome == TestOutcome.Failed) From 64d06f6a4b0e88d26938643a2fa105a47c9d2d68 Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 13:36:32 -0500 Subject: [PATCH 06/14] Fix; Resolve final xsd issues. --- src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index be842d1..9a2afd5 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -386,8 +386,14 @@ private XElement CreateTestSuiteElement(List results) { var testCaseElements = results.Select(a => this.CreateTestCaseElement(a)); - // Adding required properties element without any contents, which must occur before test cases elements. - var element = new XElement("testsuite", new XElement("properties"), testCaseElements); + // Adding required properties, system-out, and system-err elements in the correct + // positions as required by the xsd. + var element = new XElement( + "testsuite", + new XElement("properties"), + testCaseElements, + new XElement("system-out", "Junit Logger does not log standard output"), + new XElement("system-err", "Junit Logger does not log error output")); element.SetAttributeValue("name", Path.GetFileName(results.First().AssemblyPath)); From 4c139ff635186bc72b59e5646c208973f6141bfe Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 13:58:05 -0500 Subject: [PATCH 07/14] Feat: Add option to specify UTF8 without BOM. #18 --- README.md | 9 +++ .../JUnitXmlTestLogger.cs | 61 ++++++++++++++++--- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 46dfbef..ebca00c 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,15 @@ When set to default, the body will contain only the exception which is captured - FailureBodyFormat=Default - FailureBodyFormat=Verbose +#### FileEncoding + +When set to default, file encoding will be UTF-8 with BOM. Use this option if you need plain UTF-8 encoding. + +##### Allowed Values + +- FileEncoding=Utf8 +- FileEncoding=Utf8Bom (This is the default, and does not need to be specified explicitly.) + ### Saving Multiple Result Files In One Directory By default, every test project generates an xml report with the same directory and file name. The tokens {framework} and {assembly} may be placed anywhere in the directory or file names to customize the output location. This is **critical**, when multiple test reports will be written to the same directory, as in the following example. Otherwise, the files would use identical names, and the second output file written would overwrite the first. diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index 9a2afd5..b6d7871 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -36,6 +36,7 @@ public class JUnitXmlTestLogger : ITestLoggerWithParameters public const string ResultDirectoryKey = "TestRunDirectory"; public const string MethodFormatKey = "MethodFormat"; public const string FailureBodyFormatKey = "FailureBodyFormat"; + public const string FileEncodingKey = "FileEncoding"; private const string ResultStatusPassed = "Passed"; private const string ResultStatusFailed = "Failed"; @@ -81,10 +82,25 @@ public enum FailureBodyFormat Verbose } + public enum FileEncoding + { + /// + /// UTF8 + /// + UTF8, + + /// + /// UTF8 Bom + /// + UTF8Bom + } + public MethodFormat MethodFormatOption { get; private set; } = MethodFormat.Default; public FailureBodyFormat FailureBodyFormatOption { get; private set; } = FailureBodyFormat.Default; + public FileEncoding FileEncodingOption { get; private set; } = FileEncoding.UTF8Bom; + public static IEnumerable GroupTestSuites(IEnumerable suites) { var groups = suites; @@ -92,15 +108,15 @@ public static IEnumerable GroupTestSuites(IEnumerable suit while (groups.Any()) { groups = groups.GroupBy(r => - { - var name = r.FullName.SubstringBeforeDot(); - if (string.IsNullOrEmpty(name)) - { - roots.Add(r); - } - - return name; - }) + { + var name = r.FullName.SubstringBeforeDot(); + if (string.IsNullOrEmpty(name)) + { + roots.Add(r); + } + + return name; + }) .OrderBy(g => g.Key) .Where(g => !string.IsNullOrEmpty(g.Key)) .Select(g => AggregateTestSuites(g, "TestSuite", g.Key.SubstringAfterDot(), g.Key)) @@ -206,6 +222,22 @@ public void Initialize(TestLoggerEvents events, Dictionary param Console.WriteLine($"JunitXML Logger: The provided Failure Body Format '{failureFormat}' is not a recognized option. Using default"); } } + + if (parameters.TryGetValue(FileEncodingKey, out string fileEncoding)) + { + if (string.Equals(fileEncoding.Trim(), "UTF8Bom", StringComparison.OrdinalIgnoreCase)) + { + this.FileEncodingOption = FileEncoding.UTF8Bom; + } + else if (string.Equals(fileEncoding.Trim(), "UTF8", StringComparison.OrdinalIgnoreCase)) + { + this.FileEncodingOption = FileEncoding.UTF8; + } + else + { + Console.WriteLine($"JunitXML Logger: The provided File Encoding '{failureFormat}' is not a recognized option. Using default"); + } + } } /// @@ -280,9 +312,18 @@ internal void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) Directory.CreateDirectory(loggerFileDirPath); } + var settings = new XmlWriterSettings() + { + Encoding = new UTF8Encoding(this.FileEncodingOption == FileEncoding.UTF8 ? false : true), + Indent = true, + }; + using (var f = File.Create(this.outputFilePath)) { - doc.Save(f); + using (var w = XmlWriter.Create(f, settings)) + { + doc.Save(w); + } } var resultsFileMessage = string.Format(CultureInfo.CurrentCulture, "JunitXML Logger - Results File: {0}", this.outputFilePath); From 6eff7c790dcd099175de7880ca2082a89e30b723 Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 14:25:49 -0500 Subject: [PATCH 08/14] Fix: Add skipped elemnt where needed. Correct test that wasn't expecting this. --- .../JUnitXmlTestLogger.cs | 6 ++++++ .../JUnitTestLoggerAcceptanceTests.cs | 21 ++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index b6d7871..e53d472 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -495,6 +495,12 @@ private XElement CreateTestCaseElement(TestResultInfo result) testcaseElement.Add(failureElement); } + else if (result.Outcome == TestOutcome.Skipped) + { + var skippedElement = new XElement("skipped"); + + testcaseElement.Add(skippedElement); + } return testcaseElement; } diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs index 94b55b5..a6cf568 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs @@ -79,13 +79,20 @@ public void TestResultFileShouldContainTestCases() Assert.IsTrue(testcases.All(x => double.TryParse(x.Attribute("time").Value, out _))); // Check failures - Assert.AreEqual(14, testcases.Where(x => x.Descendants().Any()).Count()); - Assert.IsTrue(testcases.Where(x => x.Descendants().Any()) - .All(x => x.Descendants().Count() == 1)); - Assert.IsTrue(testcases.Where(x => x.Descendants().Any()) - .All(x => x.Descendants().First().Name.LocalName == "failure")); - Assert.IsTrue(testcases.Where(x => x.Descendants().Any()) - .All(x => x.Descendants().First().Attribute("type").Value == "failure")); + var failures = testcases + .Where(x => x.Descendants().Any(y => y.Name.LocalName == "failure")) + .ToList(); + + Assert.AreEqual(14, failures.Count()); + Assert.IsTrue(failures.All(x => x.Descendants().Count() == 1)); + Assert.IsTrue(failures.All(x => x.Descendants().First().Attribute("type").Value == "failure")); + + // Check failures + var skipps = testcases + .Where(x => x.Descendants().Any(y => y.Name.LocalName == "skipped")) + .ToList(); + + Assert.AreEqual(6, skipps.Count()); } [TestMethod] From 3ea123e4a7b5da0992f348afbd55c394ab1afba6 Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 16:43:15 -0500 Subject: [PATCH 09/14] Feat: Remove extra line breaks to cleanup failure body. --- src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs | 2 +- .../JUnitTestLoggerFormatOptionsAcceptanceTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index e53d472..8629aca 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -488,7 +488,7 @@ private XElement CreateTestCaseElement(TestResultInfo result) failureBodySB.AppendLine(result.ErrorStackTrace); - var failureElement = new XElement("failure", failureBodySB.ToString()); + var failureElement = new XElement("failure", failureBodySB.ToString().Trim()); failureElement.SetAttributeValue("type", "failure"); // TODO are there failure types? failureElement.SetAttributeValue("message", result.ErrorMessage); diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerFormatOptionsAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerFormatOptionsAcceptanceTests.cs index 0bb2f93..bdc9150 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerFormatOptionsAcceptanceTests.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerFormatOptionsAcceptanceTests.cs @@ -61,7 +61,7 @@ public void FailureBodyFormat_Verbose_ShouldStartWithMessage() var message = failure.Attribute("message").Value.Replace("\r", string.Empty).Replace("\n", string.Empty); var body = failure.Value.Replace("\r", string.Empty).Replace("\n", string.Empty); - Assert.IsTrue(body.StartsWith(message)); + Assert.IsTrue(body.Trim().StartsWith(message.Trim())); } } From c1389274edd921cf8b20dcc59c7c5c3d0e362652 Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sun, 9 Feb 2020 22:30:32 -0500 Subject: [PATCH 10/14] Feat: Prevent time from being zero. Use 0.1uS as a minmum reported value --- src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index 8629aca..f8354ae 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -472,7 +472,12 @@ private XElement CreateTestCaseElement(TestResultInfo result) testcaseElement.SetAttributeValue("name", result.Name); } - testcaseElement.SetAttributeValue("time", result.Duration.TotalSeconds.ToString("0.000000")); + // Ensure time value is never zero because gitlab treats 0 like its null. + // 0.1 micro seconds should be low enough it won't interfere with anyone + // monitoring test duration. + testcaseElement.SetAttributeValue( + "time", + Math.Max(0.0000001f, result.Duration.TotalSeconds).ToString("0.0000000")); if (result.Outcome == TestOutcome.Failed) { From c3f4fa6a81607abf27c1181e9d7d0209a499857a Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Wed, 12 Feb 2020 17:35:27 -0500 Subject: [PATCH 11/14] Feat: Improve variable/method names --- .../JUnitTestLoggerAcceptanceTests.cs | 4 ++-- .../JunitXmlValidator.cs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs index a6cf568..58df0f8 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs @@ -88,11 +88,11 @@ public void TestResultFileShouldContainTestCases() Assert.IsTrue(failures.All(x => x.Descendants().First().Attribute("type").Value == "failure")); // Check failures - var skipps = testcases + var skips = testcases .Where(x => x.Descendants().Any(y => y.Name.LocalName == "skipped")) .ToList(); - Assert.AreEqual(6, skipps.Count()); + Assert.AreEqual(6, skips.Count()); } [TestMethod] diff --git a/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs b/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs index 0f55236..4898ec1 100644 --- a/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs +++ b/test/JUnit.Xml.TestLogger.AcceptanceTests/JunitXmlValidator.cs @@ -35,12 +35,12 @@ public bool IsValid(string xml) xmlReaderSettings.Schemas.Add(schema); xmlReaderSettings.ValidationType = ValidationType.Schema; - var veh = new ValidationEventHandler(this.BooksSettingsValidationEventHandler); + var veh = new ValidationEventHandler(this.XmlValidationEventHandler); xmlReaderSettings.ValidationEventHandler += veh; - using (XmlReader books = XmlReader.Create(xmlReader, xmlReaderSettings)) + using (XmlReader reader = XmlReader.Create(xmlReader, xmlReaderSettings)) { - while (books.Read()) + while (reader.Read()) { } } @@ -50,7 +50,7 @@ public bool IsValid(string xml) return this.failures.Any() == false; } - private void BooksSettingsValidationEventHandler(object sender, ValidationEventArgs e) + private void XmlValidationEventHandler(object sender, ValidationEventArgs e) { this.failures.Add(e.Exception); } From 94eb6473cdc1ba0d58d903e82cb3586593aa31be Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Wed, 12 Feb 2020 18:16:50 -0500 Subject: [PATCH 12/14] Docs: update documentation on the junit specification and example output. --- README.md | 2 +- docs/assets/TestResults.xml | 124 ++++++++++++++++++++++++++++-------- 2 files changed, 99 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index ebca00c..9d1379b 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you're looking for `Nunit`, `Xunit` or `appveyor` loggers, visit following re ## Usage -JUnit logger can generate xml reports in the JUnit format. The JUnit format seems to be variable across different implementations, with the JUnit 5 repository referring to the [Ant Junit Format](https://github.com/windyroad/JUnit-Schema) as a de-facto standard. This project does not implement that standard exactly. Please [refer to a sample file](docs/assets/TestResults.xml) to see an example of which parts of the format have been implemented. If you find that this format is missing some feature required by your CI/CD system, please open an issue. +The JUnit Test Logger generates xml reports in the [Ant Junit Format](https://github.com/windyroad/JUnit-Schema), which the JUnit 5 repository refers to as the de-facto standard. While the generated xml complies with that schema, it does not contain values in every case. For example, the logger currently does not log any `properties`. Please [refer to a sample file](docs/assets/TestResults.xml) to see an example. If you find that the format is missing data required by your CI/CD system, please open an issue or PR. ### Default Behavior diff --git a/docs/assets/TestResults.xml b/docs/assets/TestResults.xml index 02ea554..79cb8b2 100644 --- a/docs/assets/TestResults.xml +++ b/docs/assets/TestResults.xml @@ -1,30 +1,102 @@  - - - - - - - - - at UnitTestLibrary1.MathTests.Multiply_ReturnsEight(Int32 a, Int32 b) in C:\Users\[user name]\Documents\GitHub\[solution folder]\UnitTests\MathTests.cs:line 40 - - - - - at UnitTestLibrary1.MathTests.Multiply_ReturnsEight(Int32 a, Int32 b) in C:\Users\[user name]\Documents\GitHub\[solution folder]\UnitTests\MathTests.cs:line 40 - - - - - at UnitTestLibrary1.MathTests.Multiply_ReturnsEight(Int32 a, Int32 b) in C:\Users\[user name]\Documents\GitHub\[solution folder]\UnitTests\MathTests.cs:line 40 - - - - - at UnitTestLibrary1.MathTests.Multiply_ReturnsEight(Int32 a, Int32 b) in C:\Users\[user name]\Documents\GitHub\[solution folder]\UnitTests\MathTests.cs:line 40 - - + + + + + + + + --TearDown + at JUnit.Xml.TestLogger.NetFull.Tests.FailingTearDown.TearDown() in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 175 + + + at JUnit.Xml.TestLogger.NetFull.Tests.FailingTestSetup.SetUp() in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 160 + + + + + + + at JUnit.Xml.TestLogger.NetFull.Tests.ParametrizedTestCases.TestData(Int32 x, String s) in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 225 + + + at JUnit.Xml.TestLogger.NetFull.Tests.ParametrizedTestCases.TestData(Int32 x, String s) in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 225 + + + + + + at JUnit.Xml.TestLogger.NetFull.Tests.UnitTest1.FailTest11() in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 20 + + + + + + + + + + + + + + + at JUnit.Xml.TestLogger.NetFull.Tests.UnitTest2.FailTest22() in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest1.cs:line 87 + + + + + + + + + + + + + + + --TearDown + at JUnit.Xml.TestLogger.Tests2.FailingTearDown.TearDown() in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 137 + + + at JUnit.Xml.TestLogger.Tests2.FailingTestSetup.SetUp() in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 122 + + + + + + + at JUnit.Xml.TestLogger.Tests2.ParametrizedTestCases.TestData(Int32 x, String s) in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 187 + + + at JUnit.Xml.TestLogger.Tests2.ParametrizedTestCases.TestData(Int32 x, String s) in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 187 + + + + + + at JUnit.Xml.TestLogger.Tests2.UnitTest1.FailTest11() in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 20 + + + + + + + + + at JUnit.Xml.TestLogger.Tests2.UnitTest2.FailTest22() in C:\Users\mjc82\Documents\GitHub\junit.testlogger\test\assets\JUnit.Xml.TestLogger.NetCore.Tests\UnitTest2.cs:line 49 + + + + + + + + + + Junit Logger does not log standard output + Junit Logger does not log error output \ No newline at end of file From f99c8cf0b37e06970c8ff9b29ee14a626c59073e Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sat, 15 Feb 2020 14:06:18 -0500 Subject: [PATCH 13/14] Feat: Change xml default encoding to UTF-8 --- README.md | 6 +++--- src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9d1379b..4a251ca 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,12 @@ When set to default, the body will contain only the exception which is captured #### FileEncoding -When set to default, file encoding will be UTF-8 with BOM. Use this option if you need plain UTF-8 encoding. +When set to default, file encoding will be UTF-8. Use this option if you need UTF-8 with BOM encoding. ##### Allowed Values -- FileEncoding=Utf8 -- FileEncoding=Utf8Bom (This is the default, and does not need to be specified explicitly.) +- FileEncoding=Utf8 (This is the default, and does not need to be specified explicitly.) +- FileEncoding=Utf8Bom ### Saving Multiple Result Files In One Directory diff --git a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs index f8354ae..45b2fed 100644 --- a/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs +++ b/src/JUnit.Xml.TestLogger/JUnitXmlTestLogger.cs @@ -99,7 +99,7 @@ public enum FileEncoding public FailureBodyFormat FailureBodyFormatOption { get; private set; } = FailureBodyFormat.Default; - public FileEncoding FileEncodingOption { get; private set; } = FileEncoding.UTF8Bom; + public FileEncoding FileEncodingOption { get; private set; } = FileEncoding.UTF8; public static IEnumerable GroupTestSuites(IEnumerable suites) { @@ -314,7 +314,7 @@ internal void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) var settings = new XmlWriterSettings() { - Encoding = new UTF8Encoding(this.FileEncodingOption == FileEncoding.UTF8 ? false : true), + Encoding = new UTF8Encoding(this.FileEncodingOption == FileEncoding.UTF8Bom), Indent = true, }; From 55b2bcd16c2c2f58d57a0595fc5de0dd9727b2b7 Mon Sep 17 00:00:00 2001 From: Michael Conrad Date: Sat, 15 Feb 2020 14:07:35 -0500 Subject: [PATCH 14/14] Add a changelog, pointing pointing to the releases page. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f7719c2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# Changelog + +A changelog is maintained on the releases page of the [JUnit Test Logger GitHub repository](https://github.com/spekt/junit.testlogger/).