Add a tests parsing option (useful for CI/CD)

scripts/retriveAllTests/Readme.md

@@ -0,0 +1,101 @@
+# Generate Tests YAML Script
+
+This Python script is designed to generate a YAML structure for running tests in a CI/CD pipeline. It reads .yaml files from a specified directory (assertoor-tests), applies optional filtering, and can slice the tests into groups for parallel execution.
+
+## Features
+
+- Recursive Directory Traversal: Automatically searches for .yaml test files within the assertoor-tests directory and its subdirectories.
+- Inclusion and Exclusion Filters: Allows you to specify patterns to include or exclude specific test files.
+- Grouping: Splits the tests into specified groups for parallel processing.
+
+## Usage
+To use the script, navigate to the root directory of your project and run the script using Python:
+
+### Basic Usage
+
+`python retriveAllTests.py`
+
+This will generate a YAML structure with all .yaml files found in the assertoor-tests directory and its subdirectories, excluding all.yaml.
+
+### Branch Selection
+To specify a branch other than the default (master), use the --branch option:
+
+`python retriveAllTests.py --branch "pectra"`
+
+This command will use the pectra branch instead of master to retrieve the test files.
+
+>**IMPORTANT**
+>
+>This is only a branch replacement feature in path - make sure to checkout repository on proper branch before applying this parameter.
+
+### Filtering Tests
+#### Include Specific Tests
+
+To include only certain tests based on their filenames, use the --include option:
+
+`python retriveAllTests.py --include "validator" "proposal"`
+
+This will include all files with validator or proposal in their filenames.
+
+#### Exclude Specific Tests
+
+To exclude certain tests based on their filenames, use the --exclude option:
+
+`python retriveAllTests.py --exclude "verkle" "blob"`
+
+This will exclude all files with verkle or blob in their filenames.
+
+### Slicing Tests for Parallel Execution
+
+To divide the list of tests into groups for parallel execution, use the --groups option:
+
+`python retriveAllTests.py --groups 3`
+
+This will split the tests into 3 groups.
+
+### Raw Output Usage
+
+To output the raw URLs without YAML formatting, use the --raw option:
+
+`python retriveAllTests.py --raw`
+
+This will output the URLs as plain text, one per line:
+
+```
+https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/pectra-dev/test1.yaml
+https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/verkle-dev/testA.yaml
+...
+```
+
+### Json Output Usage
+
+To output the test URLs in JSON format, use the --json option:
+
+`python retriveAllTests.py --json`
+
+This will output the URLs in a JSON format, like so:
+
+```
+{
+  "1": [
+    "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/pectra-dev/test1.yaml",
+    "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/verkle-dev/testA.yaml",
+    ...
+  ]
+}
+```
+
+## Example Output
+The script will output the assertoor_params section needed for Kurtosis network_params YAML structure in the format required for your CI/CD configuration:
+
+```
+assertoor_params:
+  tests:
+    - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/all-opcodes-test.yaml" }
+    - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/big-calldata-tx-test.yaml" }
+    - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/blob-transactions-test.yaml" }
+    - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/block-proposal-check.yaml" }
+    ...
+```
+
+If you use the --groups option, it will output separate YAML structures for each group.

scripts/retriveAllTests/retriveAllTests.py

@@ -0,0 +1,71 @@
+import os
+from pathlib import Path
+import argparse
+import math
+import json
+
+script_dir = Path(__file__).resolve().parent
+repo_dir = script_dir.parents[1]
+tests_dir = repo_dir / "assertoor-tests"
+
+def get_yaml_files(tests_dir, branch="master", include=None, exclude=None):
+    yaml_files = []
+    for root, _, files in os.walk(tests_dir):
+        for file in files:
+            if file.endswith(".yaml"):
+                full_path = Path(root) / file
+
+                if file == "all.yaml":
+                    continue
+
+                if include and not any(inc in str(full_path) for inc in include):
+                    continue
+
+                if exclude and any(exc in str(full_path) for exc in exclude):
+                    continue
+
+                raw_url = f"https://raw.githubusercontent.com/ethpandaops/assertoor-test/{branch}/{full_path.relative_to(repo_dir)}"
+                yaml_files.append(raw_url)
+
+    return yaml_files
+
+def construct_yaml_structure(yaml_files):
+    yaml_structure = "assertoor_params:\n"
+    yaml_structure += "  tests:\n"
+    for url in yaml_files:
+        yaml_structure += f"    - {{ file: \"{url}\" }}\n"
+    return yaml_structure
+
+def slice_tests(yaml_files, groups):
+    sliced_groups = []
+    group_size = math.ceil(len(yaml_files) / groups)
+    for i in range(0, len(yaml_files), group_size):
+        sliced_groups.append(yaml_files[i:i + group_size])
+    return sliced_groups
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate test URLs with optional filtering, slicing, and formatting.")
+    parser.add_argument("--branch", type=str, default="master", help="The branch name to use for constructing the raw URLs.")
+    parser.add_argument("--include", type=str, nargs='*', help="List of texts to include in file names (only include matching files).")
+    parser.add_argument("--exclude", type=str, nargs='*', help="List of texts to exclude from file names (exclude matching files).")
+    parser.add_argument("--groups", type=int, help="Number of groups to slice the tests into.", default=1)
+    parser.add_argument("--raw", action="store_true", help="Output raw URLs without YAML formatting.")
+    parser.add_argument("--json", action="store_true", help="Output URLs in JSON format.")
+
+    args = parser.parse_args()
+
+    yaml_files = get_yaml_files(tests_dir, branch=args.branch, include=args.include, exclude=args.exclude)
+    sliced_yaml_files = slice_tests(yaml_files, args.groups)
+
+    if args.json:
+        json_output = {str(idx + 1): group for idx, group in enumerate(sliced_yaml_files)}
+        print(json.dumps(json_output, indent=2))
+    else:
+        for idx, group in enumerate(sliced_yaml_files, start=1):
+            if args.raw:
+                print("\n".join(group) + "\n")
+            else:
+                print(construct_yaml_structure(group))
+
+if __name__ == "__main__":
+    main()