Merge pull request 2i2c-org#4041 from consideRatio/pr/deployer-comman…

…d-to-list-clusters

deployer: add command config get-clusters (with --provider filter flag)

deployer/README.md

@@ -196,6 +196,14 @@ For daskhubs, there is an optional check to verify that the user can scale dask
 
 Decrypts information sent to 2i2c by community representatives using [age](https://age-encryption.org/) according to instructions in [2i2c documentation](https://docs.2i2c.org/en/latest/support.html?highlight=decrypt#send-us-encrypted-content).
 
+### The `config` sub-command
+
+This deployer sub-command provides misc information.
+
+#### `config get-clusters`
+
+This function prints a sorted list of clusters, optionally filtered by the
+`--provider` flag.
 
 ### The `generate` sub-command
 
@@ -275,14 +283,14 @@ Only DaemonSet's with running pods are considered, and GPU related DaemonSets (w
 
 To run this command for all clusters, `xargs` can be used like this:
 
-    ls config/clusters | xargs -I {} deployer generate resource-allocation daemonset-requests {}
+    deployer config get-clusters | xargs -I {} deployer generate resource-allocation daemonset-requests {}
 
 ##### `generate resource-allocation instance-capacities`
 Updates `instance_capacities.yaml` with an individual cluster's running instance types' total and allocatable capacity.
 
 To run this command for all clusters, `xargs` can be used like this:
 
-    ls config/clusters | xargs -I {} deployer generate resource-allocation instance-capacities {}
+    deployer config get-clusters | xargs -I {} deployer generate resource-allocation instance-capacities {}
 
 ##### `generate resource-allocation node-info-update`
 This updates the json file `node-capacity-info.json` with info about the capacity of a node of a certain type. This file is then used for generating the resource choices.

deployer/__main__.py

@@ -1,6 +1,7 @@
 # Import the various subcommands here, they will be automatically
 # registered into the app
 import deployer.commands.cilogon  # noqa: F401
+import deployer.commands.config.get_clusters  # noqa: F401
 import deployer.commands.debug  # noqa: F401
 import deployer.commands.deployer  # noqa: F401
 import deployer.commands.exec.cloud  # noqa: F401

deployer/cli_app.py

@@ -12,6 +12,7 @@
 # Disable 'pretty' exception handling
 app = typer.Typer(pretty_exceptions_show_locals=False)
 generate_app = typer.Typer(pretty_exceptions_show_locals=False)
+config_app = typer.Typer(pretty_exceptions_show_locals=False)
 cilogon_client_app = typer.Typer(pretty_exceptions_show_locals=False)
 debug_app = typer.Typer(pretty_exceptions_show_locals=False)
 exec_app = typer.Typer(pretty_exceptions_show_locals=False)
@@ -24,6 +25,11 @@
     help="Generate various types of assets. It currently supports generating files related to billing, "
     "new dedicated clusters, helm upgrade strategies and resource allocation.",
 )
+app.add_typer(
+    config_app,
+    name="config",
+    help="Get refined information from the config folder.",
+)
 app.add_typer(
     cilogon_client_app,
     name="cilogon-client",

deployer/commands/config/get_clusters.py

@@ -0,0 +1,34 @@
+import os
+
+import typer
+from ruamel.yaml import YAML
+
+from deployer.cli_app import config_app
+from deployer.infra_components.cluster import Cluster
+from deployer.utils.file_acquisition import get_all_cluster_yaml_files
+
+# Without `pure=True`, I get an exception about str / byte issues
+yaml = YAML(typ="safe", pure=True)
+
+
+@config_app.command()
+def get_clusters(
+    provider: str = typer.Option(
+        "", help="(Optional) Filter results to clusters with this provider specified."
+    ),
+):
+    """
+    Prints all cluster names sorted alphabetically, optionally filtered by the
+    'provider' field in the cluster.yaml file.
+    """
+    cluster_names = []
+    for config_file_path in get_all_cluster_yaml_files():
+        with open(config_file_path) as f:
+            cluster = Cluster(yaml.load(f), config_file_path.parent)
+        if provider and cluster.spec["provider"] != provider:
+            continue
+        cluster_names.append(os.path.basename(config_file_path.parent))
+
+    cluster_names = sorted(cluster_names)
+    for cn in cluster_names:
+        print(cn)

deployer/commands/generate/resource_allocation/daemonset_requests.py

@@ -156,7 +156,7 @@ def daemonset_requests(
 
     To run this command for all clusters, `xargs` can be used like this:
 
-        ls config/clusters | xargs -I {} deployer generate resource-allocation daemonset-requests {}
+        deployer config get-clusters | xargs -I {} deployer generate resource-allocation daemonset-requests {}
     """
     file_path = HERE / "daemonset_requests.yaml"
     file_path.touch(exist_ok=True)

deployer/commands/generate/resource_allocation/daemonset_requests.yaml

@@ -17,7 +17,7 @@
 # This file isn't updated by automation, but can easily be updated by manually
 # running a command once for each cluster:
 #
-#     ls config/clusters | xargs -I {} deployer generate resource-allocation daemonset-requests {}
+#     deployer config get-clusters | xargs -I {} deployer generate resource-allocation daemonset-requests {}
 #
 gke:
   2i2c:

deployer/commands/generate/resource_allocation/instance_capacities.py

@@ -100,7 +100,7 @@ def instance_capacities(
 
     To run this command for all clusters, `xargs` can be used like this:
 
-        ls config/clusters | xargs -I {} deployer generate resource-allocation instance-capacities {}
+        deployer config get-clusters | xargs -I {} deployer generate resource-allocation instance-capacities {}
     """
     file_path = HERE / "instance_capacities.yaml"
     file_path.touch(exist_ok=True)

deployer/commands/generate/resource_allocation/instance_capacities.yaml

@@ -13,7 +13,7 @@
 # This file isn't updated by automation, but can easily be updated by manually
 # by running a command once for each cluster:
 #
-#     ls config/clusters | xargs -I {} deployer generate resource-allocation instance-capacities {}
+#     deployer config get-clusters | xargs -I {} deployer generate resource-allocation instance-capacities {}
 #
 # GKE instance types
 n2-highmem-2: