Working implementation using "finalizedBy"

README.md

@@ -5,13 +5,10 @@ This is a test project for a gradle build that executes both JUnit and TestNG te
 
 For testing the test execution, there is both a failing JUnit and a failing TestNG test.
 
-The build setup is currently **BROKEN:** 
+It relies on gradle's "finalizedBy" feature as well as the "TestReport" task type
+(both of which are still incubating in gradle 4.10.2).
 
-The first "./gradlew build" (initial or after "./gradlew clean") will correctly fail (because there are two failing tests).
-
-However the next invocation of "./gradlew build" will not execute the failing tests again and will incorrectly report BUILD SUCCESSFUL.
-
-**ANY IDEA?**
+Comments are welcome.
 
 Problem
 -------
@@ -42,6 +39,6 @@ Requirements
 
    People should not be forced to look at two separate test reports.
 
-4) Sane gradle behaviour must be preserved
+4) **DONE:** Sane gradle behaviour must be preserved
    * after a successful "gradle build" all tasks are up-to-date on following "gradle build" invocations unless relevant files were changed
-   * **CURRENTLY BROKEN:** Failing tests are executed again in subsequent "gradle build" invocation
+   * In case of test failures the corresponding test task is executed again in subsequent "gradle build" invocation

build.gradle

@@ -18,69 +18,67 @@ dependencies {
     testCompile 'org.testng:testng:6.14.3'
 }
 
-// Set to false in a test listener if any JUnit or TestNG test fails, 
-// so that the build can be failed after both types of tests have run
-def allTestsOk = true
-
+// count failed tests, to print a message linking to the consolidated report if needed
+project.ext.testFailureCount = 0
 // Common configuration for both JUnit and TestNG tests
 def testConfig = {
     // Make the task show up in "verification" category
     // in "gradle tasks" output and in IDEA's gradle tool window
     group = 'verification'
 
     // There is a separated task that creates a single report for both types of tests
-    test.reports.html.enabled = false
-
-    // Let the build continue, so that both types of tests can be run and 
-    // the final consolidated report can be created
-    ignoreFailures = true
-
-    // test listener that sets allTestsOk to false if any test fails
-    afterTest { TestDescriptor testDescriptor, TestResult testResult ->
-        allTestsOk &= (testResult.resultType != TestResult.ResultType.FAILURE)
-    }
+    reports.html.enabled = false
 
     testLogging {
         events TestLogEvent.FAILED // Show specific test failures in output
         exceptionFormat = TestExceptionFormat.FULL // Output full failure details
     }
+
+    // test listener that counts test failures
+    afterTest { TestDescriptor testDescriptor, TestResult testResult ->
+        if (testResult.resultType == TestResult.ResultType.FAILURE) {
+            ++project.ext.testFailureCount
+        }
+    }
+}
+
+test {
+    configure testConfig
+    description = 'Runs the JUnit tests'
+
+    finalizedBy('testNg')
 }
 
 task testNg(type: Test) {
     useTestNG()
     configure testConfig
     description = 'Runs the TestNG tests'
-}
 
-test {
-    configure testConfig
-    description = 'Runs the JUnit tests'
+    finalizedBy('testReport')
 }
 
-def testReportDir = file("$buildDir/reports/tests")
+def testReportDir = file("$buildDir/reports/test-report")
 task testReport(type: TestReport) {
     destinationDir = testReportDir
-    reportOn test, testNg
+
+    // This does *not* work:
+    //    reportOn test, testNg
+    // I guess that's because the testNg task was only executed as part of the "finalizedBy" mechanism
+    // (and not because of any "dependsOn"). So use "testResultDirs" instead:
+    testResultDirs = files("$buildDir/test-results/test/binary", "$buildDir/test-results/testNg/binary")
     group = 'verification'
     description = 'Generates a consolidated test report for both JUnit and TestNG tests'
-}
-
-check {
-    dependsOn testReport
-    finalizedBy("checkTestResults")
-}
-
-// fails the build if any JUnit or TestNG test has been run
-task checkTestResults {
-    outputs.upToDateWhen { test.state.upToDate && testNg.state.upToDate }
+
     doLast {
-        if (!allTestsOk) {
-            throw new GradleException("""
-#######################################################################################################################
-There were test failures. See console output or test report at ${new File(testReportDir, 'index.html')}.
-#######################################################################################################################
-"""
+        if (testFailureCount > 0) {
+            def failureText = (project.testFailureCount == 1 
+                    ? 'was one test failure'
+                    : "were ${project.testFailureCount} test failures"
             )
+            def msg = "There " + failureText + " - see report at file://${testReportDir}/index.html"
+            println "#" * msg.length()
+            println msg
+            println "#" * msg.length()
         }
     }
 }