From 767e1a9933d0efc0c475caad1b5dac43aae3caa8 Mon Sep 17 00:00:00 2001 From: Arvind Joshi Date: Fri, 23 Aug 2024 09:35:22 -0700 Subject: [PATCH] Fix JUnit5 reporting when tests are executed in parallel --- .../its/jiras/Surefire2252JUnit5IT.java | 61 ++++++++++++++++ .../surefire-2252-junit5-parallel-xml/pom.xml | 68 ++++++++++++++++++ .../src/test/java/pkg/domain/AxTest.java | 9 +++ .../src/test/java/pkg/domain/BxTest.java | 9 +++ .../surefire-2252-junit5-parallel/pom.xml | 68 ++++++++++++++++++ .../src/test/java/pkg/domain/AxTest.java | 9 +++ .../src/test/java/pkg/domain/BxTest.java | 9 +++ .../junitplatform/JUnitPlatformProvider.java | 19 +---- .../JUnitPlatformProviderTest.java | 72 +++++++++++++++---- 9 files changed, 294 insertions(+), 30 deletions(-) create mode 100644 surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2252JUnit5IT.java create mode 100644 surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/pom.xml create mode 100644 surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/AxTest.java create mode 100644 surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/BxTest.java create mode 100644 surefire-its/src/test/resources/surefire-2252-junit5-parallel/pom.xml create mode 100644 surefire-its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/AxTest.java create mode 100644 surefire-its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/BxTest.java diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2252JUnit5IT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2252JUnit5IT.java new file mode 100644 index 0000000000..1515e6db30 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2252JUnit5IT.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.maven.surefire.its.jiras; + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +/** + * Integration tests for SUREFIRE-2252 + */ +public class Surefire2252JUnit5IT extends SurefireJUnit4IntegrationTestCase { + @Test + public void testJUnit5() throws Exception { + unpack("surefire-2252-junit5-parallel") + .executeTest() + .verifyTextInLog( + "Using auto detected provider org.apache.maven.surefire.junitplatform.JUnitPlatformProvider") + .assertThatLogLine(containsString("Tests run: 1, Failures: 0, Errors: 0, Skipped: 0"), is(2)); + } + + @Test + public void testJUnit5Xml() { + OutputValidator validator = + unpack("surefire-2252-junit5-parallel-xml").executeTest().verifyErrorFree(2); + + validator + .getSurefireReportsFile("TEST-pkg.domain.AxTest.xml", UTF_8) + .assertContainsText("tests=\"1\"") + .assertContainsText("errors=\"0\"") + .assertContainsText("skipped=\"0\"") + .assertContainsText("failures=\"0\""); + + validator + .getSurefireReportsFile("TEST-pkg.domain.BxTest.xml", UTF_8) + .assertContainsText("tests=\"1\"") + .assertContainsText("errors=\"0\"") + .assertContainsText("skipped=\"0\"") + .assertContainsText("failures=\"0\""); + } +} diff --git a/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/pom.xml b/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/pom.xml new file mode 100644 index 0000000000..916c60104d --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/pom.xml @@ -0,0 +1,68 @@ + + + + + 4.0.0 + + org.example + surefire-2252-junit5-parallel-xml + 1.0-SNAPSHOT + + + UTF-8 + ${java.specification.version} + ${java.specification.version} + + + + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + + junit.jupiter.execution.parallel.enabled = true + junit.jupiter.execution.parallel.mode.default = same_thread + junit.jupiter.execution.parallel.mode.classes.default = concurrent + + + 2 + xml + + + + + + + diff --git a/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/AxTest.java b/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/AxTest.java new file mode 100644 index 0000000000..e5ecd5d4aa --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/AxTest.java @@ -0,0 +1,9 @@ +package pkg.domain; + +import org.junit.jupiter.api.Test; + +class AxTest { + @Test + void test() { + } +} diff --git a/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/BxTest.java b/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/BxTest.java new file mode 100644 index 0000000000..80552a95ee --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/BxTest.java @@ -0,0 +1,9 @@ +package pkg.domain; + +import org.junit.jupiter.api.Test; + +class BxTest { + @Test + void test() { + } +} diff --git a/surefire-its/src/test/resources/surefire-2252-junit5-parallel/pom.xml b/surefire-its/src/test/resources/surefire-2252-junit5-parallel/pom.xml new file mode 100644 index 0000000000..3abec8213f --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2252-junit5-parallel/pom.xml @@ -0,0 +1,68 @@ + + + + + 4.0.0 + + org.example + surefire-2252-junit5-parallel + 1.0-SNAPSHOT + + + UTF-8 + ${java.specification.version} + ${java.specification.version} + + + + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + + junit.jupiter.execution.parallel.enabled = true + junit.jupiter.execution.parallel.mode.default = same_thread + junit.jupiter.execution.parallel.mode.classes.default = concurrent + + + 2 + plain + + + + + + + diff --git a/surefire-its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/AxTest.java b/surefire-its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/AxTest.java new file mode 100644 index 0000000000..e5ecd5d4aa --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/AxTest.java @@ -0,0 +1,9 @@ +package pkg.domain; + +import org.junit.jupiter.api.Test; + +class AxTest { + @Test + void test() { + } +} diff --git a/surefire-its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/BxTest.java b/surefire-its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/BxTest.java new file mode 100644 index 0000000000..80552a95ee --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/BxTest.java @@ -0,0 +1,9 @@ +package pkg.domain; + +import org.junit.jupiter.api.Test; + +class BxTest { + @Test + void test() { + } +} diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java index 5e82554908..6c78b88e89 100644 --- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java +++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java @@ -40,7 +40,6 @@ import org.apache.maven.surefire.api.util.SurefireReflectionException; import org.apache.maven.surefire.api.util.TestsToRun; import org.apache.maven.surefire.shared.utils.StringUtils; -import org.junit.platform.engine.DiscoverySelector; import org.junit.platform.engine.Filter; import org.junit.platform.launcher.EngineFilter; import org.junit.platform.launcher.Launcher; @@ -172,25 +171,13 @@ private void invokeAllTests(TestsToRun testsToRun, RunListenerAdapter adapter) { } private void execute(TestsToRun testsToRun, RunListenerAdapter adapter) { - if (testsToRun.allowEagerReading()) { - List selectors = new ArrayList<>(); - testsToRun.iterator().forEachRemaining(c -> selectors.add(selectClass(c.getName()))); - + testsToRun.iterator().forEachRemaining(c -> { LauncherDiscoveryRequestBuilder builder = request() .filters(filters) .configurationParameters(configurationParameters) - .selectors(selectors); - + .selectors(selectClass(c.getName())); launcher.execute(builder.build(), adapter); - } else { - testsToRun.iterator().forEachRemaining(c -> { - LauncherDiscoveryRequestBuilder builder = request() - .filters(filters) - .configurationParameters(configurationParameters) - .selectors(selectClass(c.getName())); - launcher.execute(builder.build(), adapter); - }); - } + }); } private void closeLauncher() { diff --git a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java index 4b28ab18c4..8fc82f3aa9 100644 --- a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java +++ b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java @@ -199,14 +199,22 @@ public void allGivenTestsToRunAreInvoked() throws Exception { TestsToRun testsToRun = newTestsToRun(TestClass1.class, TestClass2.class); invokeProvider(provider, testsToRun); - assertThat(executionListener.summaries).hasSize(1); + assertThat(executionListener.summaries).hasSize(2); TestExecutionSummary summary = executionListener.summaries.get(0); - assertEquals(TestClass1.TESTS_FOUND + TestClass2.TESTS_FOUND, summary.getTestsFoundCount()); - assertEquals(TestClass1.TESTS_STARTED + TestClass2.TESTS_STARTED, summary.getTestsStartedCount()); - assertEquals(TestClass1.TESTS_SKIPPED + TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount()); - assertEquals(TestClass1.TESTS_SUCCEEDED + TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount()); - assertEquals(TestClass1.TESTS_ABORTED + TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount()); - assertEquals(TestClass1.TESTS_FAILED + TestClass2.TESTS_FAILED, summary.getTestsFailedCount()); + assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount()); + assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount()); + assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount()); + assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount()); + assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount()); + assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount()); + + summary = executionListener.summaries.get(1); + assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount()); + assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount()); + assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount()); + assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount()); + assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount()); + assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount()); } @Test @@ -249,6 +257,34 @@ public void singleTestClassIsInvokedLazily() throws Exception { assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount()); } + @Test + public void multipleTestClassesAreInvokedLazily() throws Exception { + Launcher launcher = LauncherFactory.create(); + JUnitPlatformProvider provider = new JUnitPlatformProvider(providerParametersMock(), launcher); + + TestPlanSummaryListener executionListener = new TestPlanSummaryListener(); + launcher.registerTestExecutionListeners(executionListener); + + invokeProvider(provider, newTestsToRunLazily(TestClass1.class, TestClass2.class)); + + assertThat(executionListener.summaries).hasSize(2); + TestExecutionSummary summary = executionListener.summaries.get(0); + assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount()); + assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount()); + assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount()); + assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount()); + assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount()); + assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount()); + + summary = executionListener.summaries.get(1); + assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount()); + assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount()); + assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount()); + assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount()); + assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount()); + assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount()); + } + @Test public void failingTestCaseAfterRerun() throws Exception { Launcher launcher = LauncherFactory.create(); @@ -682,14 +718,22 @@ public void allDiscoveredTestsAreInvokedForNullArgument() throws Exception { assertThat(report.getValue().getSourceName()).isEqualTo(TestClass2.class.getName()); assertThat(report.getValue().getName()).isNull(); - assertThat(executionListener.summaries).hasSize(1); + assertThat(executionListener.summaries).hasSize(2); TestExecutionSummary summary = executionListener.summaries.get(0); - assertEquals(TestClass1.TESTS_FOUND + TestClass2.TESTS_FOUND, summary.getTestsFoundCount()); - assertEquals(TestClass1.TESTS_STARTED + TestClass2.TESTS_STARTED, summary.getTestsStartedCount()); - assertEquals(TestClass1.TESTS_SKIPPED + TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount()); - assertEquals(TestClass1.TESTS_SUCCEEDED + TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount()); - assertEquals(TestClass1.TESTS_ABORTED + TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount()); - assertEquals(TestClass1.TESTS_FAILED + TestClass2.TESTS_FAILED, summary.getTestsFailedCount()); + assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount()); + assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount()); + assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount()); + assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount()); + assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount()); + assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount()); + + summary = executionListener.summaries.get(1); + assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount()); + assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount()); + assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount()); + assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount()); + assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount()); + assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount()); } @Test