From c32e96af795239127724c62506ee2bd1345e78da Mon Sep 17 00:00:00 2001 From: Krishnan Mahadevan Date: Sat, 6 May 2023 12:22:33 +0530 Subject: [PATCH] Generate testng-results.xml per test suite Closes #2906 --- CHANGES.txt | 5 +- .../main/java/org/testng/CommandLineArgs.java | 7 + .../src/main/java/org/testng/TestNG.java | 23 +- .../testng/reporters/AbstractXmlReporter.java | 284 ++++++++++++++++++ .../testng/reporters/PerSuiteXMLReporter.java | 55 ++++ .../org/testng/reporters/XMLReporter.java | 206 +------------ .../test/reports/PerSuiteXMLReporterTest.java | 107 +++++++ .../reports/issue2906/SampleOneTestCase.java | 18 ++ .../issue2906/SampleThreeTestCase.java | 18 ++ .../reports/issue2906/SampleTwoTestCase.java | 18 ++ testng-core/src/test/resources/testng.xml | 1 + 11 files changed, 547 insertions(+), 195 deletions(-) create mode 100644 testng-core/src/main/java/org/testng/reporters/AbstractXmlReporter.java create mode 100644 testng-core/src/main/java/org/testng/reporters/PerSuiteXMLReporter.java create mode 100644 testng-core/src/test/java/test/reports/PerSuiteXMLReporterTest.java create mode 100644 testng-core/src/test/java/test/reports/issue2906/SampleOneTestCase.java create mode 100644 testng-core/src/test/java/test/reports/issue2906/SampleThreeTestCase.java create mode 100644 testng-core/src/test/java/test/reports/issue2906/SampleTwoTestCase.java diff --git a/CHANGES.txt b/CHANGES.txt index c160c9e95..70307ae27 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,7 @@ Current -New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen) -New: GITHUB-2907: Added assertListContains and assertListContainsObject methods to check if specific object present in List (Dmytro Budym) +Fixed: GITHUB-2906: Generate testng-results.xml per test suite (Krishnan Mahadevan) +New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen) +New: GITHUB-2907: Added assertListContains and assertListContainsObject methods to check if specific object present in List (Dmytro Budym) Fixed: GITHUB-2888: Skipped Tests with DataProvider appear as failed (Joaquin Moreira) Fixed: GITHUB-2884: Discrepancies with DataProvider and Retry of failed tests (Krishnan Mahadevan) Fixed: GITHUB-2879: Test listeners specified in parent testng.xml file are not included in testng-failed.xml file (Krishnan Mahadevan) diff --git a/testng-core/src/main/java/org/testng/CommandLineArgs.java b/testng-core/src/main/java/org/testng/CommandLineArgs.java index f696de1d0..bad8b7c0a 100644 --- a/testng-core/src/main/java/org/testng/CommandLineArgs.java +++ b/testng-core/src/main/java/org/testng/CommandLineArgs.java @@ -267,4 +267,11 @@ public class CommandLineArgs { names = PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE, description = "Should TestNG consider failures in Data Providers as test failures.") public Boolean propagateDataProviderFailureAsTestFailure = false; + + public static final String GENERATE_RESULTS_PER_SUITE = "-generateResultsPerSuite"; + + @Parameter( + names = GENERATE_RESULTS_PER_SUITE, + description = "Should TestNG consider failures in Data Providers as test failures.") + public Boolean generateResultsPerSuite = false; } diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java index 339b1b6bf..d68e9c773 100644 --- a/testng-core/src/main/java/org/testng/TestNG.java +++ b/testng-core/src/main/java/org/testng/TestNG.java @@ -54,6 +54,7 @@ import org.testng.reporters.EmailableReporter2; import org.testng.reporters.FailedReporter; import org.testng.reporters.JUnitReportReporter; +import org.testng.reporters.PerSuiteXMLReporter; import org.testng.reporters.SuiteHTMLReporter; import org.testng.reporters.VerboseReporter; import org.testng.reporters.XMLReporter; @@ -811,6 +812,7 @@ public List getSuiteListeners() { private Boolean m_preserveOrder = XmlSuite.DEFAULT_PRESERVE_ORDER; private Boolean m_groupByInstances; + private boolean m_generateResultsPerSuite = false; private IConfiguration m_configuration; @@ -829,6 +831,10 @@ public void setExecutorFactoryClass(String clazzName) { this.m_executorFactory = createExecutorFactoryInstanceUsing(clazzName); } + public void setGenerateResultsPerSuite(boolean generateResultsPerSuite) { + this.m_generateResultsPerSuite = generateResultsPerSuite; + } + private IExecutorFactory createExecutorFactoryInstanceUsing(String clazzName) { Class cls = ClassHelper.forName(clazzName); Object instance = m_objectFactory.newInstance(cls); @@ -934,7 +940,11 @@ private void initializeDefaultListeners() { addReporter(SuiteHTMLReporter.class); addReporter(Main.class); addReporter(FailedReporter.class); - addReporter(XMLReporter.class); + if (m_generateResultsPerSuite) { + addReporter(PerSuiteXMLReporter.class); + } else { + addReporter(XMLReporter.class); + } if (RuntimeBehavior.useOldTestNGEmailableReporter()) { addReporter(EmailableReporter.class); } else if (RuntimeBehavior.useEmailableReporter()) { @@ -1449,6 +1459,9 @@ protected void configure(CommandLineArgs cla) { Optional.ofNullable(cla.propagateDataProviderFailureAsTestFailure) .ifPresent(value -> propagateDataProviderFailureAsTestFailure()); setReportAllDataDrivenTestsAsSkipped(cla.includeAllDataDrivenTestsWhenSkipping); + + Optional.ofNullable(cla.generateResultsPerSuite).ifPresent(this::setGenerateResultsPerSuite); + if (cla.verbose != null) { setVerbose(cla.verbose); } @@ -1744,6 +1757,14 @@ public void configure(Map cmdLineArgs) { result.dependencyInjectorFactoryClass = dependencyInjectorFactoryClass; } + result.ignoreMissedTestNames = + Boolean.parseBoolean( + cmdLineArgs.getOrDefault(CommandLineArgs.IGNORE_MISSED_TEST_NAMES, false).toString()); + + result.generateResultsPerSuite = + Boolean.parseBoolean( + cmdLineArgs.getOrDefault(CommandLineArgs.GENERATE_RESULTS_PER_SUITE, false).toString()); + configure(result); } diff --git a/testng-core/src/main/java/org/testng/reporters/AbstractXmlReporter.java b/testng-core/src/main/java/org/testng/reporters/AbstractXmlReporter.java new file mode 100644 index 000000000..3f93cf880 --- /dev/null +++ b/testng-core/src/main/java/org/testng/reporters/AbstractXmlReporter.java @@ -0,0 +1,284 @@ +package org.testng.reporters; + +import java.io.File; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.testng.IReporter; +import org.testng.ISuite; +import org.testng.ISuiteResult; +import org.testng.ITestContext; +import org.testng.ITestNGMethod; +import org.testng.ITestResult; +import org.testng.Reporter; +import org.testng.internal.Utils; +import org.testng.util.TimeUtils; + +public abstract class AbstractXmlReporter implements IReporter, ICustomizeXmlReport { + + private final XMLReporterConfig config = new XMLReporterConfig(); + + public String fileName() { + return RuntimeBehavior.getDefaultFileNameForXmlReports(); + } + + @Override + public XMLReporterConfig getConfig() { + return config; + } + + @Override + public void addCustomTagsFor(XMLStringBuffer xmlBuffer, ITestResult testResult) {} + + protected final void writeReporterOutput(XMLStringBuffer xmlBuffer) { + writeReporterOutput(xmlBuffer, Reporter.getOutput()); + } + + protected final void writeReporterOutput(XMLStringBuffer xmlBuffer, List output) { + // TODO: Cosmin - maybe a element isn't indicated for each line + xmlBuffer.push(XMLReporterConfig.TAG_REPORTER_OUTPUT); + for (String line : output) { + if (line != null) { + xmlBuffer.push(XMLReporterConfig.TAG_LINE); + xmlBuffer.addCDATA(line); + xmlBuffer.pop(); + } + } + xmlBuffer.pop(); + } + + protected final void writeSuite(XMLStringBuffer rootBuffer, ISuite suite) { + switch (config.getFileFragmentationLevel()) { + case XMLReporterConfig.FF_LEVEL_NONE: + writeSuiteToBuffer(rootBuffer, suite); + break; + case XMLReporterConfig.FF_LEVEL_SUITE: + case XMLReporterConfig.FF_LEVEL_SUITE_RESULT: + File suiteFile = referenceSuite(rootBuffer, suite); + writeSuiteToFile(suiteFile, suite); + break; + default: + throw new AssertionError("Unexpected value: " + config.getFileFragmentationLevel()); + } + } + + private void writeSuiteToFile(File suiteFile, ISuite suite) { + XMLStringBuffer xmlBuffer = new XMLStringBuffer(); + writeSuiteToBuffer(xmlBuffer, suite); + File parentDir = suiteFile.getParentFile(); + suiteFile.getParentFile().mkdirs(); + if (parentDir.exists() || suiteFile.getParentFile().exists()) { + Utils.writeUtf8File(parentDir.getAbsolutePath(), fileName(), xmlBuffer.toXML()); + } + } + + private File referenceSuite(XMLStringBuffer xmlBuffer, ISuite suite) { + String relativePath = suite.getName() + File.separatorChar + fileName(); + File suiteFile = new File(config.getOutputDirectory(), relativePath); + Properties attrs = new Properties(); + attrs.setProperty(XMLReporterConfig.ATTR_URL, relativePath); + xmlBuffer.addEmptyElement(XMLReporterConfig.TAG_SUITE, attrs); + return suiteFile; + } + + private void writeSuiteToBuffer(XMLStringBuffer xmlBuffer, ISuite suite) { + xmlBuffer.push(XMLReporterConfig.TAG_SUITE, getSuiteAttributes(suite)); + writeSuiteGroups(xmlBuffer, suite); + + Map results = suite.getResults(); + XMLSuiteResultWriter suiteResultWriter = new XMLSuiteResultWriter(config, this); + for (Map.Entry result : results.entrySet()) { + suiteResultWriter.writeSuiteResult(xmlBuffer, result.getValue()); + } + + xmlBuffer.pop(); + } + + private Set getUniqueMethodSet(Collection methods) { + return new LinkedHashSet<>(methods); + } + + private void writeSuiteGroups(XMLStringBuffer xmlBuffer, ISuite suite) { + xmlBuffer.push(XMLReporterConfig.TAG_GROUPS); + Map> methodsByGroups = suite.getMethodsByGroups(); + for (Map.Entry> entry : methodsByGroups.entrySet()) { + Properties groupAttrs = new Properties(); + groupAttrs.setProperty(XMLReporterConfig.ATTR_NAME, entry.getKey()); + xmlBuffer.push(XMLReporterConfig.TAG_GROUP, groupAttrs); + Set groupMethods = getUniqueMethodSet(entry.getValue()); + for (ITestNGMethod groupMethod : groupMethods) { + Properties methodAttrs = new Properties(); + methodAttrs.setProperty(XMLReporterConfig.ATTR_NAME, groupMethod.getMethodName()); + methodAttrs.setProperty(XMLReporterConfig.ATTR_METHOD_SIG, groupMethod.toString()); + methodAttrs.setProperty(XMLReporterConfig.ATTR_CLASS, groupMethod.getRealClass().getName()); + xmlBuffer.addEmptyElement(XMLReporterConfig.TAG_METHOD, methodAttrs); + } + xmlBuffer.pop(); + } + xmlBuffer.pop(); + } + + private Properties getSuiteAttributes(ISuite suite) { + Properties props = new Properties(); + props.setProperty(XMLReporterConfig.ATTR_NAME, suite.getName()); + + // Calculate the duration + Map results = suite.getResults(); + Date minStartDate = new Date(); + Date maxEndDate = null; + // TODO: We could probably optimize this in order not to traverse this twice + for (Map.Entry result : results.entrySet()) { + ITestContext testContext = result.getValue().getTestContext(); + Date startDate = testContext.getStartDate(); + Date endDate = testContext.getEndDate(); + if (minStartDate.after(startDate)) { + minStartDate = startDate; + } + if (maxEndDate == null || maxEndDate.before(endDate)) { + maxEndDate = endDate != null ? endDate : startDate; + } + } + // The suite could be completely empty + if (maxEndDate == null) { + maxEndDate = minStartDate; + } + setDurationAttributes(config, props, minStartDate, maxEndDate); + return props; + } + + protected static void setDurationAttributes( + XMLReporterConfig config, Properties attributes, Date minStartDate, Date maxEndDate) { + + String startTime = + TimeUtils.formatTimeInLocalOrSpecifiedTimeZone( + minStartDate.getTime(), config.getTimestampFormat()); + String endTime = + TimeUtils.formatTimeInLocalOrSpecifiedTimeZone( + maxEndDate.getTime(), config.getTimestampFormat()); + long duration = maxEndDate.getTime() - minStartDate.getTime(); + + attributes.setProperty(XMLReporterConfig.ATTR_STARTED_AT, startTime); + attributes.setProperty(XMLReporterConfig.ATTR_FINISHED_AT, endTime); + attributes.setProperty(XMLReporterConfig.ATTR_DURATION_MS, Long.toString(duration)); + } + + protected final Properties writeSummaryCount(Count count, XMLStringBuffer rootBuffer) { + Properties p = new Properties(); + p.put("passed", count.passed); + p.put("failed", count.failed); + p.put("skipped", count.skipped); + if (count.retried > 0) { + p.put("retried", count.retried); + } + p.put("ignored", count.ignored); + p.put("total", count.total()); + return p; + } + + protected final Count computeCountForSuite(ISuite s) { + int passed = 0; + int failed = 0; + int skipped = 0; + int retried = 0; + int ignored = 0; + for (ISuiteResult sr : s.getResults().values()) { + ITestContext testContext = sr.getTestContext(); + passed += testContext.getPassedTests().size(); + failed += testContext.getFailedTests().size(); + int retriedPerTest = 0; + int skippedPerTest = 0; + for (ITestResult result : testContext.getSkippedTests().getAllResults()) { + if (result.wasRetried()) { + retriedPerTest++; + } else { + skippedPerTest++; + } + } + skipped += skippedPerTest; + retried += retriedPerTest; + ignored += testContext.getExcludedMethods().stream().filter(ITestNGMethod::isTest).count(); + } + return Count.Builder.builder() + .withPassed(passed) + .withFailed(failed) + .withSkipped(skipped) + .withRetried(retried) + .withIgnored(ignored) + .build(); + } + + public static class Count { + private int passed; + private int failed; + private int skipped; + private int retried; + private int ignored; + + public int total() { + return passed + failed + skipped + retried + ignored; + } + + public void add(Count count) { + this.passed += count.passed; + this.failed += count.failed; + this.skipped += count.skipped; + this.retried += count.retried; + this.ignored += count.retried; + } + + private Count(Builder builder) { + passed = builder.passed; + failed = builder.failed; + skipped = builder.skipped; + retried = builder.retried; + ignored = builder.ignored; + } + + public static final class Builder { + private int passed; + private int failed; + private int skipped; + private int retried; + private int ignored; + + private Builder() {} + + public static Builder builder() { + return new Builder(); + } + + public Builder withPassed(int val) { + passed = val; + return this; + } + + public Builder withFailed(int val) { + failed = val; + return this; + } + + public Builder withSkipped(int val) { + skipped = val; + return this; + } + + public Builder withRetried(int val) { + retried = val; + return this; + } + + public Builder withIgnored(int val) { + ignored = val; + return this; + } + + public Count build() { + return new Count(this); + } + } + } +} diff --git a/testng-core/src/main/java/org/testng/reporters/PerSuiteXMLReporter.java b/testng-core/src/main/java/org/testng/reporters/PerSuiteXMLReporter.java new file mode 100644 index 000000000..e1b2f0216 --- /dev/null +++ b/testng-core/src/main/java/org/testng/reporters/PerSuiteXMLReporter.java @@ -0,0 +1,55 @@ +package org.testng.reporters; + +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import org.testng.ISuite; +import org.testng.ISuiteResult; +import org.testng.ITestContext; +import org.testng.ITestResult; +import org.testng.Reporter; +import org.testng.internal.Utils; +import org.testng.xml.XmlSuite; + +/** The main entry for the XML generation operation */ +public class PerSuiteXMLReporter extends AbstractXmlReporter { + + @Override + public void generateReport( + List xmlSuites, List suites, String outputDirectory) { + if (Utils.isStringEmpty(getConfig().getOutputDirectory())) { + getConfig().setOutputDirectory(outputDirectory); + } + + for (ISuite s : suites) { + Count count = computeCountForSuite(s); + XMLStringBuffer rootBuffer = new XMLStringBuffer(); + Properties p = writeSummaryCount(count, rootBuffer); + rootBuffer.push(XMLReporterConfig.TAG_TESTNG_RESULTS, p); + writeReporterOutput(rootBuffer, getOutput(s)); + writeSuite(rootBuffer, s); + rootBuffer.pop(); + String dir = getConfig().getOutputDirectory() + "/" + s.getName(); + Utils.writeUtf8File(dir, fileName(), rootBuffer, null /* no prefix */); + } + } + + private List getOutput(ISuite iSuite) { + return iSuite.getResults().values().stream() + .map(ISuiteResult::getTestContext) + .flatMap(each -> results(each).stream()) + .flatMap(each -> Reporter.getOutput(each).stream()) + .collect(Collectors.toList()); + } + + private Set results(ITestContext context) { + Set result = new HashSet<>(context.getPassedConfigurations().getAllResults()); + result.addAll(context.getFailedConfigurations().getAllResults()); + result.addAll(context.getPassedTests().getAllResults()); + result.addAll(context.getFailedTests().getAllResults()); + result.addAll(context.getFailedButWithinSuccessPercentageTests().getAllResults()); + return result; + } +} diff --git a/testng-core/src/main/java/org/testng/reporters/XMLReporter.java b/testng-core/src/main/java/org/testng/reporters/XMLReporter.java index bc0d2cc68..4f88c24ff 100644 --- a/testng-core/src/main/java/org/testng/reporters/XMLReporter.java +++ b/testng-core/src/main/java/org/testng/reporters/XMLReporter.java @@ -1,229 +1,51 @@ package org.testng.reporters; -import java.io.File; -import java.util.Collection; import java.util.Date; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Properties; -import java.util.Set; -import org.testng.IReporter; import org.testng.ISuite; -import org.testng.ISuiteResult; -import org.testng.ITestContext; -import org.testng.ITestNGMethod; -import org.testng.ITestResult; -import org.testng.Reporter; import org.testng.internal.Utils; -import org.testng.util.TimeUtils; import org.testng.xml.XmlSuite; /** The main entry for the XML generation operation */ -public class XMLReporter implements IReporter, ICustomizeXmlReport { - - private final XMLReporterConfig config = new XMLReporterConfig(); - private XMLStringBuffer rootBuffer; +public class XMLReporter extends AbstractXmlReporter { @Override public void generateReport( List xmlSuites, List suites, String outputDirectory) { - if (Utils.isStringEmpty(config.getOutputDirectory())) { - config.setOutputDirectory(outputDirectory); + if (Utils.isStringEmpty(getConfig().getOutputDirectory())) { + getConfig().setOutputDirectory(outputDirectory); } // Calculate passed/failed/skipped - int passed = 0; - int failed = 0; - int skipped = 0; - int ignored = 0; - int retried = 0; + Count count = Count.Builder.builder().build(); for (ISuite s : suites) { - Map suiteResults = s.getResults(); - for (ISuiteResult sr : suiteResults.values()) { - ITestContext testContext = sr.getTestContext(); - passed += testContext.getPassedTests().size(); - failed += testContext.getFailedTests().size(); - int retriedPerTest = 0; - int skippedPerTest = 0; - for (ITestResult result : testContext.getSkippedTests().getAllResults()) { - if (result.wasRetried()) { - retriedPerTest++; - } else { - skippedPerTest++; - } - } - skipped += skippedPerTest; - retried += retriedPerTest; - ignored += testContext.getExcludedMethods().stream().filter(ITestNGMethod::isTest).count(); - } + count.add(computeCountForSuite(s)); } - rootBuffer = new XMLStringBuffer(); - Properties p = new Properties(); - p.put("passed", passed); - p.put("failed", failed); - p.put("skipped", skipped); - if (retried > 0) { - p.put("retried", retried); - } - p.put("ignored", ignored); - p.put("total", passed + failed + skipped + ignored + retried); + XMLStringBuffer rootBuffer = new XMLStringBuffer(); + Properties p = writeSummaryCount(count, rootBuffer); rootBuffer.push(XMLReporterConfig.TAG_TESTNG_RESULTS, p); writeReporterOutput(rootBuffer); for (ISuite suite : suites) { - writeSuite(suite); + writeSuite(rootBuffer, suite); } rootBuffer.pop(); - Utils.writeUtf8File(config.getOutputDirectory(), fileName(), rootBuffer, null /* no prefix */); - } - - @Override - public void addCustomTagsFor(XMLStringBuffer xmlBuffer, ITestResult testResult) {} - - public String fileName() { - return RuntimeBehavior.getDefaultFileNameForXmlReports(); - } - - private void writeReporterOutput(XMLStringBuffer xmlBuffer) { - // TODO: Cosmin - maybe a element isn't indicated for each line - xmlBuffer.push(XMLReporterConfig.TAG_REPORTER_OUTPUT); - List output = Reporter.getOutput(); - for (String line : output) { - if (line != null) { - xmlBuffer.push(XMLReporterConfig.TAG_LINE); - xmlBuffer.addCDATA(line); - xmlBuffer.pop(); - } - } - xmlBuffer.pop(); - } - - private void writeSuite(ISuite suite) { - switch (config.getFileFragmentationLevel()) { - case XMLReporterConfig.FF_LEVEL_NONE: - writeSuiteToBuffer(rootBuffer, suite); - break; - case XMLReporterConfig.FF_LEVEL_SUITE: - case XMLReporterConfig.FF_LEVEL_SUITE_RESULT: - File suiteFile = referenceSuite(rootBuffer, suite); - writeSuiteToFile(suiteFile, suite); - break; - default: - throw new AssertionError("Unexpected value: " + config.getFileFragmentationLevel()); - } - } - - private void writeSuiteToFile(File suiteFile, ISuite suite) { - XMLStringBuffer xmlBuffer = new XMLStringBuffer(); - writeSuiteToBuffer(xmlBuffer, suite); - File parentDir = suiteFile.getParentFile(); - suiteFile.getParentFile().mkdirs(); - if (parentDir.exists() || suiteFile.getParentFile().exists()) { - Utils.writeUtf8File(parentDir.getAbsolutePath(), fileName(), xmlBuffer.toXML()); - } - } - - private File referenceSuite(XMLStringBuffer xmlBuffer, ISuite suite) { - String relativePath = suite.getName() + File.separatorChar + fileName(); - File suiteFile = new File(config.getOutputDirectory(), relativePath); - Properties attrs = new Properties(); - attrs.setProperty(XMLReporterConfig.ATTR_URL, relativePath); - xmlBuffer.addEmptyElement(XMLReporterConfig.TAG_SUITE, attrs); - return suiteFile; - } - - private void writeSuiteToBuffer(XMLStringBuffer xmlBuffer, ISuite suite) { - xmlBuffer.push(XMLReporterConfig.TAG_SUITE, getSuiteAttributes(suite)); - writeSuiteGroups(xmlBuffer, suite); - - Map results = suite.getResults(); - XMLSuiteResultWriter suiteResultWriter = new XMLSuiteResultWriter(config, this); - for (Map.Entry result : results.entrySet()) { - suiteResultWriter.writeSuiteResult(xmlBuffer, result.getValue()); - } - - xmlBuffer.pop(); - } - - private void writeSuiteGroups(XMLStringBuffer xmlBuffer, ISuite suite) { - xmlBuffer.push(XMLReporterConfig.TAG_GROUPS); - Map> methodsByGroups = suite.getMethodsByGroups(); - for (Map.Entry> entry : methodsByGroups.entrySet()) { - Properties groupAttrs = new Properties(); - groupAttrs.setProperty(XMLReporterConfig.ATTR_NAME, entry.getKey()); - xmlBuffer.push(XMLReporterConfig.TAG_GROUP, groupAttrs); - Set groupMethods = getUniqueMethodSet(entry.getValue()); - for (ITestNGMethod groupMethod : groupMethods) { - Properties methodAttrs = new Properties(); - methodAttrs.setProperty(XMLReporterConfig.ATTR_NAME, groupMethod.getMethodName()); - methodAttrs.setProperty(XMLReporterConfig.ATTR_METHOD_SIG, groupMethod.toString()); - methodAttrs.setProperty(XMLReporterConfig.ATTR_CLASS, groupMethod.getRealClass().getName()); - xmlBuffer.addEmptyElement(XMLReporterConfig.TAG_METHOD, methodAttrs); - } - xmlBuffer.pop(); - } - xmlBuffer.pop(); - } - - private Properties getSuiteAttributes(ISuite suite) { - Properties props = new Properties(); - props.setProperty(XMLReporterConfig.ATTR_NAME, suite.getName()); - - // Calculate the duration - Map results = suite.getResults(); - Date minStartDate = new Date(); - Date maxEndDate = null; - // TODO: We could probably optimize this in order not to traverse this twice - for (Map.Entry result : results.entrySet()) { - ITestContext testContext = result.getValue().getTestContext(); - Date startDate = testContext.getStartDate(); - Date endDate = testContext.getEndDate(); - if (minStartDate.after(startDate)) { - minStartDate = startDate; - } - if (maxEndDate == null || maxEndDate.before(endDate)) { - maxEndDate = endDate != null ? endDate : startDate; - } - } - // The suite could be completely empty - if (maxEndDate == null) { - maxEndDate = minStartDate; - } - addDurationAttributes(config, props, minStartDate, maxEndDate); - return props; + Utils.writeUtf8File( + getConfig().getOutputDirectory(), fileName(), rootBuffer, null /* no prefix */); } /** - * Add started-at, finished-at and duration-ms attributes to the <suite> tag - * + * @deprecated - This method stands deprecated as of TestNG 7.8.0 Add started-at, + * finished-at and duration-ms attributes to the <suite> tag * @param config The reporter config * @param attributes The properties * @param minStartDate The minimum start date * @param maxEndDate The maximum end date */ + @Deprecated public static void addDurationAttributes( XMLReporterConfig config, Properties attributes, Date minStartDate, Date maxEndDate) { - - String startTime = - TimeUtils.formatTimeInLocalOrSpecifiedTimeZone( - minStartDate.getTime(), config.getTimestampFormat()); - String endTime = - TimeUtils.formatTimeInLocalOrSpecifiedTimeZone( - maxEndDate.getTime(), config.getTimestampFormat()); - long duration = maxEndDate.getTime() - minStartDate.getTime(); - - attributes.setProperty(XMLReporterConfig.ATTR_STARTED_AT, startTime); - attributes.setProperty(XMLReporterConfig.ATTR_FINISHED_AT, endTime); - attributes.setProperty(XMLReporterConfig.ATTR_DURATION_MS, Long.toString(duration)); - } - - private Set getUniqueMethodSet(Collection methods) { - return new LinkedHashSet<>(methods); - } - - @Override - public XMLReporterConfig getConfig() { - return config; + setDurationAttributes(config, attributes, minStartDate, maxEndDate); } } diff --git a/testng-core/src/test/java/test/reports/PerSuiteXMLReporterTest.java b/testng-core/src/test/java/test/reports/PerSuiteXMLReporterTest.java new file mode 100644 index 000000000..9b53ef24b --- /dev/null +++ b/testng-core/src/test/java/test/reports/PerSuiteXMLReporterTest.java @@ -0,0 +1,107 @@ +package test.reports; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import org.testng.CommandLineArgs; +import org.testng.TestNG; +import org.testng.annotations.Test; +import org.testng.xml.XmlSuite; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; +import test.SimpleBaseTest; +import test.reports.issue2906.SampleOneTestCase; +import test.reports.issue2906.SampleThreeTestCase; +import test.reports.issue2906.SampleTwoTestCase; + +public class PerSuiteXMLReporterTest extends SimpleBaseTest { + + @Test(description = "GITHUB-2906") + public void ensurePerSuiteGenerationHappens() throws Exception { + runTest(testng -> testng.setGenerateResultsPerSuite(true)); + } + + @Test(description = "GITHUB-2906") + public void ensurePerSuiteGenerationHappensWithEnabledViaMap() throws Exception { + runTest( + testng -> { + Map map = new HashMap<>(); + map.put(CommandLineArgs.GENERATE_RESULTS_PER_SUITE, "true"); + testng.configure(map); + }); + } + + private void runTest(Consumer tweaker) throws Exception { + XmlSuite parentSuite = createXmlSuite("parent_suite"); + XmlSuite childSuite1 = createChildSuite("child_one", SampleOneTestCase.class); + XmlSuite childSuite2 = createChildSuite("child_two", SampleTwoTestCase.class); + XmlSuite childSuite3 = createChildSuite("child_three", SampleThreeTestCase.class); + parentSuite.getChildSuites().addAll(Arrays.asList(childSuite1, childSuite2, childSuite3)); + + File parentDir = createDirInTempDir("parent_suite"); + TestNG testng = create(parentDir.toPath(), parentSuite); + testng.setUseDefaultListeners(true); + tweaker.accept(testng); + testng.run(); + runVerificationsForSuite(parentDir, childSuite1, "firstPassingTestCase() ran"); + runVerificationsForSuite(parentDir, childSuite2, "secondPassingTestCase() ran"); + runVerificationsForSuite(parentDir, childSuite3, "thirdPassingTestCase() ran"); + } + + private static void runVerificationsForSuite(File parentDir, XmlSuite xmlSuite, String output) + throws ParserConfigurationException, IOException, SAXException, XPathExpressionException { + File dir = new File(parentDir, xmlSuite.getName()); + File resultsFile = new File(dir, "testng-results.xml"); + assertThat(resultsFile).exists(); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(resultsFile); + XPath xPath = XPathFactory.newInstance().newXPath(); + Node node = (Node) xPath.compile("/testng-results").evaluate(doc, XPathConstants.NODE); + int ignored = Integer.parseInt(node.getAttributes().getNamedItem("ignored").getNodeValue()); + int total = Integer.parseInt(node.getAttributes().getNamedItem("total").getNodeValue()); + int passed = Integer.parseInt(node.getAttributes().getNamedItem("passed").getNodeValue()); + int failed = Integer.parseInt(node.getAttributes().getNamedItem("failed").getNodeValue()); + assertThat(ignored).isZero(); + assertThat(total).isEqualTo(2); + assertThat(passed).isEqualTo(1); + assertThat(failed).isEqualTo(1); + String text = + (String) + xPath + .compile("/testng-results/reporter-output/line") + .evaluate(doc, XPathConstants.STRING); + text = text.stripLeading().stripTrailing().replaceAll("\\r\\n|\\r|\\n", ""); + assertThat(text).isEqualTo(output); + node = + (Node) xPath.compile("/testng-results/suite/test/class").evaluate(doc, XPathConstants.NODE); + String expected = xmlSuite.getTests().get(0).getClasses().get(0).getName(); + assertThat(node.getAttributes().getNamedItem("name").getNodeValue()).isEqualTo(expected); + NodeList nodes = + (NodeList) + xPath + .compile("/testng-results/suite/test/class/test-method") + .evaluate(doc, XPathConstants.NODESET); + assertThat(nodes.getLength()).isEqualTo(2); + } + + private static XmlSuite createChildSuite(String suffix, Class testClass) { + XmlSuite xmlSuite = createXmlSuite("suite_" + suffix); + createXmlTest(xmlSuite, "test_" + suffix, testClass); + return xmlSuite; + } +} diff --git a/testng-core/src/test/java/test/reports/issue2906/SampleOneTestCase.java b/testng-core/src/test/java/test/reports/issue2906/SampleOneTestCase.java new file mode 100644 index 000000000..819d23027 --- /dev/null +++ b/testng-core/src/test/java/test/reports/issue2906/SampleOneTestCase.java @@ -0,0 +1,18 @@ +package test.reports.issue2906; + +import org.testng.Assert; +import org.testng.Reporter; +import org.testng.annotations.Test; + +public class SampleOneTestCase { + + @Test + public void firstPassingTestCase() { + Reporter.log("firstPassingTestCase() ran"); + } + + @Test + public void firstFailingTestCase() { + Assert.fail(); + } +} diff --git a/testng-core/src/test/java/test/reports/issue2906/SampleThreeTestCase.java b/testng-core/src/test/java/test/reports/issue2906/SampleThreeTestCase.java new file mode 100644 index 000000000..900a3759c --- /dev/null +++ b/testng-core/src/test/java/test/reports/issue2906/SampleThreeTestCase.java @@ -0,0 +1,18 @@ +package test.reports.issue2906; + +import org.testng.Assert; +import org.testng.Reporter; +import org.testng.annotations.Test; + +public class SampleThreeTestCase { + + @Test + public void thirdPassingTestCase() { + Reporter.log("thirdPassingTestCase() ran"); + } + + @Test + public void thirdFailingTestCase() { + Assert.fail(); + } +} diff --git a/testng-core/src/test/java/test/reports/issue2906/SampleTwoTestCase.java b/testng-core/src/test/java/test/reports/issue2906/SampleTwoTestCase.java new file mode 100644 index 000000000..3cb8ed384 --- /dev/null +++ b/testng-core/src/test/java/test/reports/issue2906/SampleTwoTestCase.java @@ -0,0 +1,18 @@ +package test.reports.issue2906; + +import org.testng.Assert; +import org.testng.Reporter; +import org.testng.annotations.Test; + +public class SampleTwoTestCase { + + @Test + public void secondPassingTestCase() { + Reporter.log("secondPassingTestCase() ran"); + } + + @Test + public void secondFailingTestCase() { + Assert.fail(); + } +} diff --git a/testng-core/src/test/resources/testng.xml b/testng-core/src/test/resources/testng.xml index 66df4cc7b..7013764a2 100644 --- a/testng-core/src/test/resources/testng.xml +++ b/testng-core/src/test/resources/testng.xml @@ -113,6 +113,7 @@ +