Organize extends tests (#2375)

* Add GITHUB_REF_NAME to tests

* Reorder config test folders

* Add local remote config mix test

* Rename folder

* Fix linter

* Fix linter

* Fix tests

.automation/test/mega-linter-config-test/local_extends_recurse/base.local.mega-linter.yml

@@ -0,0 +1,2 @@
+FILTER_REGEX_INCLUDE: "(base-local)"
+SHOW_ELAPSED_TIME: true

.automation/test/mega-linter-config-test/local_extends_recurse/local.mega-linter.yml

@@ -0,0 +1,3 @@
+EXTENDS: base.local.mega-linter.yml
+FILTER_REGEX_INCLUDE: "(local)"
+SHOW_ELAPSED_TIME: false

.automation/test/mega-linter-config-test/local_remote_extends_recurse/local.remote.mega-linter.yml

@@ -0,0 +1,3 @@
+EXTENDS: base.mega-linter.yml
+DEFAULT_BRANCH: dev
+LOG_LEVEL: DEBUG

.automation/test/mega-linter-config-test/remote_extends/base.mega-linter.yml

@@ -0,0 +1,5 @@
+EXTENDS:
+  - "https://raw.githubusercontent.com/oxsecurity/megalinter/main/.automation/test/mega-linter-config-test/extension1.mega-linter.yml"
+  - "https://raw.githubusercontent.com/oxsecurity/megalinter/main/.automation/test/mega-linter-config-test/extension2.mega-linter.yml"
+FILTER_REGEX_INCLUDE: "(base)"
+SHOW_ELAPSED_TIME: true

.github/workflows/deploy-BETA-linters.yml

@@ -241,7 +241,7 @@ jobs:
         run: |
           TEST_KEYWORDS_TO_USE_UPPER="${{ matrix.linter }}"
           TEST_KEYWORDS_TO_USE="${TEST_KEYWORDS_TO_USE_UPPER,,}"
-          docker run -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAIL=detailed -e GITHUB_SHA=${GITHUB_SHA} -e PAT="${{secrets.PAT}}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v ${GITHUB_WORKSPACE}:/tmp/lint oxsecurity/megalinter-only-${{ matrix.linter }}:beta
+          docker run -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAIL=detailed -e GITHUB_SHA=${GITHUB_SHA} -e GITHUB_REF_NAME=${GITHUB_REF_NAME} -e PAT="${{secrets.PAT}}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v ${GITHUB_WORKSPACE}:/tmp/lint oxsecurity/megalinter-only-${{ matrix.linter }}:beta
         timeout-minutes: 30
 
       ##############################################

.github/workflows/deploy-DEV-linters.yml

@@ -225,7 +225,7 @@ jobs:
         run: |
           TEST_KEYWORDS_TO_USE_UPPER="${{ matrix.linter }}"
           TEST_KEYWORDS_TO_USE="${TEST_KEYWORDS_TO_USE_UPPER,,}"
-          docker run -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAIL=detailed -e GITHUB_SHA=${GITHUB_SHA} -e PAT="${{secrets.PAT}}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v ${GITHUB_WORKSPACE}:/tmp/lint oxsecurity/megalinter-only-${{ matrix.linter }}:${{ needs.prepare.outputs.tag }}
+          docker run -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAIL=detailed -e GITHUB_SHA=${GITHUB_SHA} -e GITHUB_REF_NAME=${GITHUB_REF_NAME} -e PAT="${{secrets.PAT}}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v ${GITHUB_WORKSPACE}:/tmp/lint oxsecurity/megalinter-only-${{ matrix.linter }}:${{ needs.prepare.outputs.tag }}
         timeout-minutes: 30
 
       ##############################################

.github/workflows/deploy-DEV.yml

@@ -207,7 +207,7 @@ jobs:
                 TEST_KEYWORDS_TO_USE=""
             fi
           fi
-          docker run $CI_ENV -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAIL=detailed -e GITHUB_SHA=${GITHUB_SHA} -e PAT="${{secrets.PAT}}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v ${GITHUB_WORKSPACE}:/tmp/lint oxsecurity/megalinter:${{steps.image_tag.outputs.tag}}
+          docker run $CI_ENV -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAIL=detailed -e GITHUB_SHA=${GITHUB_SHA} -e GITHUB_REF_NAME=${GITHUB_REF_NAME} -e PAT="${{secrets.PAT}}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v ${GITHUB_WORKSPACE}:/tmp/lint oxsecurity/megalinter:${{steps.image_tag.outputs.tag}}
         timeout-minutes: 60
 
       #####################################

.github/workflows/deploy-RELEASE-linters.yml

@@ -203,7 +203,7 @@ jobs:
         run: |
           TEST_KEYWORDS_TO_USE_UPPER="${{ matrix.linter }}"
           TEST_KEYWORDS_TO_USE="${TEST_KEYWORDS_TO_USE_UPPER,,}"
-          docker run -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAIL=detailed -e GITHUB_SHA=${GITHUB_SHA} -e PAT="${{secrets.PAT}}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v ${GITHUB_WORKSPACE}:/tmp/lint oxsecurity/megalinter-only-${{ matrix.linter }}:${{ github.event.release.tag_name }}
+          docker run -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAIL=detailed -e GITHUB_SHA=${GITHUB_SHA} -e GITHUB_REF_NAME=${GITHUB_REF_NAME} -e PAT="${{secrets.PAT}}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v ${GITHUB_WORKSPACE}:/tmp/lint oxsecurity/megalinter-only-${{ matrix.linter }}:${{ github.event.release.tag_name }}
         timeout-minutes: 30
 
       ##############################################

megalinter/tests/test_megalinter/config_test.py

@@ -3,18 +3,25 @@
 Unit tests for Megalinter class
 
 """
+import glob
 import os
+import re
 import unittest
 
+from git import Repo
+
 from megalinter import config
 from megalinter.constants import ML_REPO
+from megalinter.tests.test_megalinter.helpers import utilstest
 from megalinter.utils import REPO_HOME_DEFAULT
 
 
 class config_test(unittest.TestCase):
+    branch = os.environ.get("GITHUB_REF_NAME", "main")
+
     test_folder = (
         f"https://raw.githubusercontent.com/{ML_REPO}/"
-        "main/.automation/test/mega-linter-config-test/"
+        f"{branch}/.automation/test/mega-linter-config-test/"
     )
 
     def setUp(self):
@@ -33,20 +40,30 @@ def tearDown(self):
         config.delete()
 
     def test_remote_config_success(self):
-        remote_config = self.test_folder + "custom.mega-linter.yml"
+        changed_files = self.replace_branch_in_input_files()
+        remote_config = self.test_folder + "remote/custom.mega-linter.yml"
         os.environ["MEGALINTER_CONFIG"] = remote_config
         config.init_config()
         self.assertEqual("(custom)", config.get("FILTER_REGEX_INCLUDE"))
+        self.restore_branch_in_input_files(changed_files)
 
     def test_remote_config_error(self):
+        changed_files = self.replace_branch_in_input_files()
         remote_config = self.test_folder + "custom.mega-linter-not-existing.yml"
         try:
             os.environ["MEGALINTER_CONFIG"] = remote_config
             config.init_config()
         except Exception as e:
-            self.assertIn("http", str(e))
+            self.assertRegex(str(e), (
+                "Unable to retrieve config file "
+                r"https://.*/\.automation/test/mega-linter-config-test/"
+                r"custom\.mega-linter-not-existing\.yml"
+            ))
+        finally:
+            self.restore_branch_in_input_files(changed_files)
 
     def test_local_config_extends_success(self):
+        changed_files = self.replace_branch_in_input_files()
         local_config = "local.mega-linter.yml"
         os.environ["MEGALINTER_CONFIG"] = local_config
         config.init_config(
@@ -57,11 +74,15 @@ def test_local_config_extends_success(self):
             + "test"
             + os.path.sep
             + "mega-linter-config-test"
+            + os.path.sep
+            + "local_extends"
         )
         self.assertEqual("(local)", config.get("FILTER_REGEX_INCLUDE"))
         self.assertEqual("false", config.get("SHOW_ELAPSED_TIME"))
+        self.restore_branch_in_input_files(changed_files)
 
     def test_local_config_extends_recurse_success(self):
+        changed_files = self.replace_branch_in_input_files()
         local_config = "recurse.mega-linter.yml"
         os.environ["MEGALINTER_CONFIG"] = local_config
         config.init_config(
@@ -72,13 +93,17 @@ def test_local_config_extends_recurse_success(self):
             + "test"
             + os.path.sep
             + "mega-linter-config-test"
+            + os.path.sep
+            + "local_extends_recurse"
         )
         self.assertEqual("(local)", config.get("FILTER_REGEX_INCLUDE"))
         self.assertEqual("false", config.get("SHOW_ELAPSED_TIME"))
         self.assertEqual("dev", config.get("DEFAULT_BRANCH"))
         self.assertEqual("DEBUG", config.get("LOG_LEVEL"))
+        self.restore_branch_in_input_files(changed_files)
 
     def test_local_config_extends_error(self):
+        changed_files = self.replace_branch_in_input_files()
         local_config = "local-error.mega-linter.yml"
         os.environ["MEGALINTER_CONFIG"] = local_config
         try:
@@ -93,31 +118,66 @@ def test_local_config_extends_error(self):
             )
         except Exception as e:
             self.assertIn("No such file or directory", str(e))
+        finally:
+            self.restore_branch_in_input_files(changed_files)
 
     def test_remote_config_extends_success(self):
-        remote_config = self.test_folder + "base.mega-linter.yml"
+        changed_files = self.replace_branch_in_input_files()
+        remote_config = self.test_folder + "remote_extends/base.mega-linter.yml"
         os.environ["MEGALINTER_CONFIG"] = remote_config
         config.init_config()
         self.assertEqual("(base)", config.get("FILTER_REGEX_INCLUDE"))
         self.assertEqual("(extension2)", config.get("FILTER_REGEX_EXCLUDE"))
         self.assertEqual("true", config.get("SHOW_ELAPSED_TIME"))
+        self.restore_branch_in_input_files(changed_files)
 
     def test_remote_config_extends_success_2(self):
-        remote_config = self.test_folder + "base2.mega-linter.yml"
+        changed_files = self.replace_branch_in_input_files()
+        remote_config = self.test_folder + "remote_extends_2/base2.mega-linter.yml"
         os.environ["MEGALINTER_CONFIG"] = remote_config
         config.init_config()
         self.assertEqual("(base)", config.get("FILTER_REGEX_INCLUDE"))
         self.assertEqual("(extension2)", config.get("FILTER_REGEX_EXCLUDE"))
         self.assertEqual("true", config.get("SHOW_ELAPSED_TIME"))
+        self.restore_branch_in_input_files(changed_files)
 
     def test_remote_config_extends_error(self):
-        remote_config = self.test_folder + "base-error.mega-linter.yml"
+        changed_files = self.replace_branch_in_input_files()
+        remote_config = self.test_folder + "remote_extends_error/base-error.mega-linter.yml"
         os.environ["MEGALINTER_CONFIG"] = remote_config
         try:
             os.environ["MEGALINTER_CONFIG"] = remote_config
             config.init_config()
         except Exception as e:
-            self.assertIn("http", str(e))
+            self.assertRegex(str(e), (
+                "Unable to retrieve EXTENDS config file "
+                r"https://.*/\.automation/test/mega-linter-config-test/"
+                r"remote_extends_error/base-error\.mega-linter\.yml"
+            ))
+        finally:
+            self.restore_branch_in_input_files(changed_files)
+
+    def test_local_remote_config_extends_recurse_success(self):
+        changed_files = self.replace_branch_in_input_files()
+        local_config = "local.remote.mega-linter.yml"
+        os.environ["MEGALINTER_CONFIG"] = local_config
+        config.init_config(
+            REPO_HOME_DEFAULT
+            + os.path.sep
+            + ".automation"
+            + os.path.sep
+            + "test"
+            + os.path.sep
+            + "mega-linter-config-test"
+            + os.path.sep
+            + "local_remote_extends_recurse"
+        )
+        self.assertEqual("(base)", config.get("FILTER_REGEX_INCLUDE"))
+        self.assertEqual("(extension2)", config.get("FILTER_REGEX_EXCLUDE"))
+        self.assertEqual("true", config.get("SHOW_ELAPSED_TIME"))
+        self.assertEqual("dev", config.get("DEFAULT_BRANCH"))
+        self.assertEqual("DEBUG", config.get("LOG_LEVEL"))
+        self.restore_branch_in_input_files(changed_files)
 
     def test_list_of_obj_as_env_var(self):
         os.environ[
@@ -127,3 +187,55 @@ def test_list_of_obj_as_env_var(self):
         pre_commands = config.get_list("PRE_COMMANDS", [])
         del os.environ["PRE_COMMANDS"]
         self.assertTrue(len(pre_commands) > 0, "PRE_COMMANDS not loaded from ENV var")
+
+    def replace_branch_in_input_files(self):
+        root = (
+            ".automation"
+            + os.path.sep
+            + "test"
+            + os.path.sep
+            + "mega-linter-config-test"
+        )
+
+        branch = os.environ.get("GITHUB_REF_NAME", "main")
+
+        search_glob_pattern = root.replace("\\", "/") + "/**/*"
+
+        regex = r"(/oxsecurity/megalinter/)(main)(/\.automation)"
+
+        list = []
+
+        for file in glob.iglob(search_glob_pattern, recursive=True):
+            file_name = os.path.basename(file)
+            if (".yml" not in file_name):
+                continue
+
+            match = False
+
+            with open(file, "r", encoding="utf-8") as f:
+                file_content = f.read()
+
+                if re.search(regex, file_content):
+                    file_content = re.sub(
+                        regex,
+                        rf"\1{branch}\3",
+                        file_content
+                    )
+
+                    match = True
+
+            if match:
+                with open(file, "w") as f:
+                    f.write(file_content)
+
+                list.append(file)
+
+        return list
+
+    def restore_branch_in_input_files(self, files):
+        repo = Repo(os.path.realpath(utilstest.REPO_HOME))
+
+        for file in files:
+            repo.index.checkout(
+                [os.path.join(os.path.realpath(utilstest.REPO_HOME), file)], force=True
+            )