report

#!/usr/local/bin/env -S scala-cli -Dlog4j.configurationFile=scripts/log4j.properties shebang --quiet

//> using scala 3.6.2
//> using option -source:future

object ClocReport:

  // sometimes there's an extra "[info]" in the line, so we have to be careful
  val Regex = """\[([^\]]+)\] (?:\[info\] )?\*\* COMMUNITY BUILD LINE COUNT: (\d+)""".r

  def apply(log: io.Source): Unit =
    val results = collection.mutable.Map.empty[String, Int].withDefaultValue(0)
    for case Regex(project, count) <- log.getLines do
      results(project) += count.toInt
    println("Lines of Scala code recompiled during this run only:")
    for (project, sum) <- results.toSeq.sortBy(_._2).reverse do
      println(f"$sum%9d $project")
    println(f"${results.values.sum}%9d TOTAL")

object SuccessReport:

  // sample inputs:
  //   [info] Project foo-bar-baz---------------: DID NOT RUN (stuck on broken dependencies: frob, akka-grue, zorch)
  //   [info] Project utest---------------------: FAILED (MatchError: d48a6cde+20180920-1730 (of class java.lang.St...)
  // regex features used:
  //   \w    = word character
  //   ?:    = not a capturing group
  //   (?!-) = negative lookahead -- next character is not "-"
  val Regex =
    """\[info\] Project ((?:\w|-(?!-))+)-*: ([^\(]+) \((?:stuck on broken dependencies: )?(.*)\)""".r

  val requiresJdk11Plus = Set[String](
    "airframe",
    "doobie",       // JDK 8 support dropped (January 2024)
    "fastscala",
    "fs2",
    "jsoniter-scala", // JDK 8 support dropped (February 2022)
    "metrics-scala", // JDK 8 support dropped (August 2023)
    "mdoc", // JDK 8 support dropped (October 2023)
    "parboiled", // JDK 8 support dropped (March 2022)
    "pass4s",
    "play-file-watch", // JDK 8 support dropped (October 2022)
    "playframework", // JDK 8 support dropped (September 2022)
    "requests-scala", // JDK 8 support dropped (June 2024)
    "scaffeine", // they seem to be on a caffeine version that requires JDK 11
    "scalachess",
    "scalafmt",  // uses mdoc, which dropped JDK 8
    "slick",  // JDK 8 support dropped (June 2024)
    "sttp", // uses java.net.http
    "unfiltered", // they seem to be on a jetty version that requires JDK 11+
  )

  val requiresJdk17Plus = Set[String](
    "americium",  // some test code uses `String#translateEscapes`
    "cachecontrol", // JDK 11 support dropped (July 2024)
    "metals", // JDK 11 support dropped (August 2024)
    "play-ws", // JDK 11 support dropped (July 2024)
    "scala-commons",
    "scastie",
    "shapeless-java-records", // inherently requires JDK 15+
    "twirl", // JDK 11 support dropped (July 2024)
  )

  val requiresJdk21Plus = Set[String](
    "scalafx",  // uses -release:21
  )

  val jdk11Failures = Set[String](
    "scala-debugger", // "object FieldInfo is not a member of package sun.reflect"
    "scala-refactoring", // needs scala/bug#11125 workaround?
  )

  val jdk17Failures = Set[String](
    "akka", // needs newer sbt-osgi; not sure about status w/r/t JEP 403
    "finagle", // Unrecognized VM option 'AggressiveOpts'
    "mockito-scala", // reflection-related test failures
    "scrooge", // Unrecognized VM option 'AggressiveOpts'
    "twitter-util", // Unrecognized VM option 'AggressiveOpts'
  )

  val jdk21Failures = Set[String](
    "kafka",         // cannot override final member Thread#threadId
    "ssl-config",    // compilation errors involving sun.security.x509
  )

  val jdk22Failures = Set[String](
    "twirl",         // Unsupported class file major version 66
    "playframework", // Java 22 (66) is not supported by the current version of Byte Buddy
  )

  val jdk23Failures = Set[String](
    "cats-effect-tutorial", // import conflict involving java.io.IO
    "fetch",                // import conflict involving java.io.IO
    "fpinscala",            // import conflict involving java.io.IO
    "monix",                // import conflict involving java.io.IO
    "zinc",                 // Unsupported class file major version 67
  )

  val expectedToFail: Set[String] =
    sys.env.get("jvmVersion") match
      case Some("8") =>
        requiresJdk11Plus ++ requiresJdk17Plus ++ requiresJdk21Plus
      case Some("11") =>
        jdk11Failures ++ requiresJdk17Plus ++ requiresJdk21Plus
      case Some("17") =>
        jdk11Failures ++ jdk17Failures ++ requiresJdk21Plus
      case Some("21") =>
        jdk11Failures ++ jdk17Failures ++ jdk21Failures
      case Some("22") =>
        jdk11Failures ++ jdk17Failures ++ jdk21Failures ++ jdk22Failures
      case Some(_) =>
        jdk11Failures ++ jdk17Failures ++ jdk21Failures ++ jdk22Failures ++ jdk23Failures
      case None => // we're not running on Jenkins, apparently
        Set()

  def apply(log: io.Source): Option[Int] =
    val lines = log.getLines.dropWhile(!_.contains("---==  Execution Report ==---"))
    var success, failed, didNotRun = 0
    val unexpectedSuccesses = collection.mutable.Buffer[String]()
    val unexpectedFailures = collection.mutable.Buffer[String]()
    val blockerCounts = collection.mutable.Map[String, Int]()
    for case Regex(name, status, blockers) <- lines do
      status match
        case "EXTRACTION FAILED" =>
          success = -1000
        case "SUCCESS" =>
          success += 1
          if expectedToFail(name) then
            unexpectedSuccesses += name
        case "FAILED" =>
          failed += 1
          if !expectedToFail(name) then
            unexpectedFailures += name
        case "DID NOT RUN" =>
          didNotRun += 1
          for blocker <- blockers.split(',').map(_.trim) do
            blockerCounts(blocker) = 1 + blockerCounts.getOrElse(blocker, 0)
    if success < 0 then return None
    val total = success + failed + didNotRun
    println(s"SUCCEEDED: $success")
    val sortedFailures =
      unexpectedFailures.sortBy(blockerCounts.withDefaultValue(0)).reverse
    if unexpectedFailures.nonEmpty then
      val uf = sortedFailures.mkString(",")
      println(s"FAILURES (UNEXPECTED): $uf")
    if didNotRun > 0 then
      val blockers =
        blockerCounts.toList.sortBy(_._2).reverse
          .collect: (blocker, count) =>
            s"$blocker ($count)"
          .mkString(", ")
      println(s"BLOCKING DOWNSTREAM: $blockers")
    if unexpectedSuccesses.nonEmpty then
      val us = unexpectedSuccesses.mkString(",")
      println(s"SUCCESSES (UNEXPECTED): $us")
    println(s"FAILED: $failed")
    println(s"BLOCKED, DID NOT RUN: $didNotRun")
    println(s"TOTAL: $total")
    for url <- Option(System.getenv("BUILD_URL")) do
      if unexpectedFailures.nonEmpty then
        println()
        for failed <- sortedFailures do
          println(s"""<a href="${url}artifact/logs/$failed-build.log">$failed</a>""")
    if success == 0 then
      Some(1)
    else
      Some(unexpectedFailures.size)

object SplitLog:

  val BeginDependencies = """\[info\] ---== Dependency Information ===---""".r
  val EndDependencies = """\[info\] ---== ()End Dependency Information ===---""".r
  val BeginExtract = """\[([^\]]+)\] --== Extracting dependencies for .+ ==--""".r
  val EndExtract = """\[([^\]]+)\] --== End Extracting dependencies for .+ ==--""".r
  val BeginBuild = """\[([^\]]+)\] --== Building .+ ==--""".r
  val EndBuild = """\[([^\]]+)\] --== End Building .+ ==--""".r

  def apply(log: io.Source): Unit =
    val dir = java.io.File("logs")
    dir.mkdirs()
    val lines = log.getLines
    while lines.hasNext do
      lines.next match
        case BeginDependencies() =>
          slurp(lines, makeWriter("logs/_dependencies.log"), EndDependencies)
        case BeginExtract(name) =>
          slurp(lines, makeWriter(s"logs/$name-extract.log"), EndExtract)
        case BeginBuild(name) =>
          slurp(lines, makeWriter(s"logs/$name-build.log"), EndBuild)
        case _ =>

  import java.io.PrintWriter

  def makeWriter(path: String): PrintWriter =
    import java.io.*
    val file = File(path)
    val foStream = FileOutputStream(file, false) // false = overwrite, don't append
    val osWriter = OutputStreamWriter(foStream)
    PrintWriter(osWriter)

  import scala.util.matching.Regex, annotation.tailrec

  def slurp(lines: Iterator[String], writer: java.io.PrintWriter, sentinel: Regex): Unit =
    @tailrec def iterate(): Unit =
      if lines.hasNext then
        lines.next match
          case sentinel(_) =>
            writer.close()
          case line =>
            writer.println(line)
            iterate()
      else
        writer.close()
    iterate()

// main
def log = io.Source.fromFile(args(0))
println("<pre>")
ClocReport(log)
val unexpectedFailureCount = SuccessReport(log)
println("</pre>")
SplitLog(log)
sys.exit(unexpectedFailureCount.getOrElse(1))