Update flow

.github/logToCs.py

@@ -1,4 +1,5 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
+# pylint: disable=invalid-name
 """
 Convert a log to CheckStyle format.
 
@@ -48,10 +49,13 @@
 import argparse
 import re
 import sys
-import xml.etree.ElementTree as ET
+import xml.etree.ElementTree as ET  # nosec
 
 
 def convert_to_checkstyle(messages):
+    """
+    Convert provided message to CheckStyle format.
+    """
     root = ET.Element("checkstyle")
     for message in messages:
         fields = parse_message(message)
@@ -60,23 +64,45 @@ def convert_to_checkstyle(messages):
     return ET.tostring(root, encoding="utf-8").decode("utf-8")
 
 
+def convert_text_to_checkstyle(text):
+    """
+    Convert provided message to CheckStyle format.
+    """
+    root = ET.Element("checkstyle")
+    for fields in parse_file(text):
+        if fields:
+            add_error_entry(root, **fields)
+    return ET.tostring(root, encoding="utf-8").decode("utf-8")
+
+
 ANY_REGEX = r".*?"
 FILE_REGEX = r"\s*(?P<file_name>\S.*?)\s*?"
+EOL_REGEX = r"[\r\n]"
 LINE_REGEX = r"\s*(?P<line>\d+?)\s*?"
 COLUMN_REGEX = r"\s*(?P<column>\d+?)\s*?"
 SEVERITY_REGEX = r"\s*(?P<severity>error|warning|notice|style|info)\s*?"
 MSG_REGEX = r"\s*(?P<message>.+?)\s*?"
+MULTILINE_MSG_REGEX = r"\s*(?P<message>(?:.|.[\r\n])+)"
 # cpplint confidence index
 CONFIDENCE_REGEX = r"\s*\[(?P<confidence>\d+)\]\s*?"
 
 
 # List of message patterns, add more specific patterns earlier in the list
 # Creating patterns by using constants makes them easier to define and read.
 PATTERNS = [
-    # ESLint (JavaScript Linter), RoboCop
-    # path/to/file.js:10:2: Some linting issue
-    # path/to/file.rb:10:5: Style/Indentation: Incorrect indentation detected
-    # path/to/script.sh:10:1: SC2034: Some shell script issue
+    # beautysh
+    #  File ftp.sh: error: "esac" before "case" in line 90.
+    re.compile(
+        f"^File {FILE_REGEX}:{SEVERITY_REGEX}:"
+        f" {MSG_REGEX} in line {LINE_REGEX}.$"
+    ),
+    # beautysh
+    #  File socks4echo.sh: error: indent/outdent mismatch: -2.
+    re.compile(f"^File {FILE_REGEX}:{SEVERITY_REGEX}: {MSG_REGEX}$"),
+    # ESLint (JavaScript Linter), RoboCop, shellcheck
+    #  path/to/file.js:10:2: Some linting issue
+    #  path/to/file.rb:10:5: Style/Indentation: Incorrect indentation detected
+    #  path/to/script.sh:10:1: SC2034: Some shell script issue
     re.compile(f"^{FILE_REGEX}:{LINE_REGEX}:{COLUMN_REGEX}: {MSG_REGEX}$"),
     # Cpplint default output:
     #           '%s:%s:  %s  [%s] [%d]\n'
@@ -99,7 +125,10 @@ def convert_to_checkstyle(messages):
     re.compile(f"^{FILE_REGEX}:{LINE_REGEX}: {MSG_REGEX}$"),
     # Shellcheck:
     # In script.sh line 76:
-    re.compile(f"^In {FILE_REGEX} line {LINE_REGEX}:({MSG_REGEX})?$"),
+    re.compile(
+        f"^In {FILE_REGEX} line {LINE_REGEX}:{EOL_REGEX}?"
+        f"({MULTILINE_MSG_REGEX})?{EOL_REGEX}{EOL_REGEX}"
+    ),
 ]
 
 # Severities available in CodeSniffer report format
@@ -108,17 +137,78 @@ def convert_to_checkstyle(messages):
 SEVERITY_ERROR = "error"
 
 
+def strip_ansi(text: str):
+    """
+    Strip ANSI escape sequences from string (colors, etc)
+    """
+    return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", text)
+
+
+def parse_file(text):
+    """
+    Parse all messages in a file
+
+    Returns the fields in a dict.
+    """
+    # regex required to allow same group names
+    import regex  # pylint: disable=import-outside-toplevel
+
+    patterns = [pattern.pattern for pattern in PATTERNS]
+    # patterns = [PATTERNS[0].pattern]
+
+    full_regex = "(?:(?:" + (")|(?:".join(patterns)) + "))"
+    results = []
+
+    for fields in regex.finditer(
+        full_regex, strip_ansi(text), regex.MULTILINE
+    ):
+        if not fields:
+            continue
+        result = fields.groupdict()
+
+        if len(result) == 0:
+            continue
+        severity = result.get("severity", None)
+        confidence = result.pop("confidence", None)
+
+        if confidence is not None:
+            # Convert confidence level of cpplint
+            # to warning, etc.
+            confidence = int(confidence)
+
+            if confidence <= 1:
+                severity = SEVERITY_NOTICE
+            elif confidence >= 5:
+                severity = SEVERITY_ERROR
+            else:
+                severity = SEVERITY_WARNING
+
+        if severity is None:
+            severity = SEVERITY_ERROR
+        else:
+            severity = severity.lower()
+
+        if severity in ["info", "style"]:
+            severity = SEVERITY_NOTICE
+
+        result["severity"] = severity
+
+        results.append(result)
+
+    return results
+
+
 def parse_message(message):
     """
     Parse message until it matches a pattern.
 
     Returns the fields in a dict.
     """
     for pattern in PATTERNS:
-        m = pattern.match(message)
-        if not m:
+        fields = pattern.match(message)
+        if not fields:
             continue
-        result = m.groupdict()
+        result = fields.groupdict()
         if len(result) == 0:
             continue
 
@@ -159,6 +249,9 @@ def add_error_entry(  # pylint: disable=too-many-arguments
     message=None,
     source=None,
 ):
+    """
+    Add error information to the CheckStyle output being created.
+    """
     file_element = find_or_create_file_element(root, file_name)
     error_element = ET.SubElement(file_element, "error")
     error_element.set("severity", severity)
@@ -174,6 +267,9 @@ def add_error_entry(  # pylint: disable=too-many-arguments
 
 
 def find_or_create_file_element(root, file_name):
+    """
+    Find/create file element in XML document tree.
+    """
     for file_element in root.findall("file"):
         if file_element.get("name") == file_name:
             return file_element
@@ -183,6 +279,9 @@ def find_or_create_file_element(root, file_name):
 
 
 def main():
+    """
+    Parse the script arguments and get the conversion done.
+    """
     parser = argparse.ArgumentParser(
         description="Convert messages to Checkstyle XML format."
     )
@@ -210,14 +309,17 @@ def main():
 
     if args.input == "-" and args.input_named:
         with open(args.input_named, encoding="utf_8") as input_file:
-            messages = input_file.readlines()
+            text = input_file.read()
     elif args.input != "-":
         with open(args.input, encoding="utf_8") as input_file:
-            messages = input_file.readlines()
+            text = input_file.read()
     else:
-        messages = sys.stdin.readlines()
+        text = sys.stdin.read()
 
-    checkstyle_xml = convert_to_checkstyle(messages)
+    try:
+        checkstyle_xml = convert_text_to_checkstyle(text)
+    except ImportError:
+        checkstyle_xml = convert_to_checkstyle(re.split(r"[\r\n]+", text))
 
     if args.output == "-" and args.output_named:
         with open(args.output_named, "w", encoding="utf_8") as output_file:

.github/workflows/pre-commit.yml

@@ -18,8 +18,8 @@ jobs:
         if: false
         with:
           cache: pip
-          python-version: "3.12.1"
-      - run: python -m pip install pre-commit
+          python-version: 3.12.1
+      - run: python -m pip install pre-commit regex
       - uses: actions/cache/restore@v3
         with:
           path: ~/.cache/pre-commit/
@@ -28,21 +28,31 @@ jobs:
       - name: Run pre-commit hooks
         run: |
           set -o pipefail
+          pre-commit gc
           pre-commit run --show-diff-on-failure --color=always --all-files | tee ${RAW_LOG}
       - name: Convert Raw Log to CheckStyle format
         if: ${{ failure() }}
         run: |
-          ${LOG_TO_CS} ${RAW_LOG} ${CS_XML}
+          python ${LOG_TO_CS} ${RAW_LOG} ${CS_XML}
       - name: Annotate Source Code with Messages
         uses: staabm/annotate-pull-request-from-checkstyle-action@v1
         if: ${{ failure() }}
         with:
           files: ${{ env.CS_XML }}
-          notices-as-warnings: true # optional
-          prepend-filename: true # optional
+          notices-as-warnings: true  # optional
+          prepend-filename: true  # optional
       - uses: actions/cache/save@v3
         if: ${{ always() }}
         with:
           path: ~/.cache/pre-commit/
           key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml')
             }}
+      - name: Provide log as artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: precommit-logs
+          path: |
+            ${{ env.RAW_LOG }}
+            ${{ env.CS_XML }}
+          retention-days: 2

.pre-commit-config.yaml

@@ -120,6 +120,7 @@ repos:
           - pyflakes
           - paho-mqtt
           - aiohttp
+          - pylint
         args:
           - --reports=no
   - repo: https://github.com/pre-commit/mirrors-mypy