diff --git a/deployer/README.md b/deployer/README.md index 95b14fa0b6..8628bc08ba 100644 --- a/deployer/README.md +++ b/deployer/README.md @@ -384,6 +384,30 @@ setup or an outage), or when taking down a hub. All these commands take a cluster and hub name as parameters, and perform appropriate authentication before performing their function. +#### `exec hub` + +This subcommand gives you an interactive shell on the hub pod itself, so you +can poke around to see what's going on. Particularly useful if you want to peek +at the hub db with the `sqlite` command. + +#### `exec homes` + +This subcommand gives you a shell with the home directories of all the +users on the given hub in the given cluster mounted under `/home`. +Very helpful when doing (rare) manual operations on user home directories, +such as renames. + +When you exit the shell, the temporary pod spun up is removed. + +#### `exec root-homes` + +Similar to `exec homes` but mounts the _entire_ NFS filesystem to `/root-homes`. +You can optionally mount a secondary NFS share if required, which is useful when migrating data across servers. + +#### `exec aws` + +This sub-command can exec into a shell with appropriate AWS credentials (including MFA). + #### `exec debug` This sub-command is useful for debugging. @@ -410,28 +434,6 @@ in our 2i2c cluster that can be accessed via this command, and speeds up image b Once you run this command, run `export DOCKER_HOST=tcp://localhost:23760` in another terminal to use the faster remote docker daemon. -#### `exec shell` -This exec sub-command can be used to acquire a shell in various places of the infrastructure. - -##### `exec shell hub` - -This subcommand gives you an interactive shell on the hub pod itself, so you -can poke around to see what's going on. Particularly useful if you want to peek -at the hub db with the `sqlite` command. - -##### `exec shell homes` - -This subcommand gives you a shell with the home directories of all the -users on the given hub in the given cluster mounted under `/home`. -Very helpful when doing (rare) manual operations on user home directories, -such as renames. - -When you exit the shell, the temporary pod spun up is removed. - -##### `exec shell aws` - -This sub-command can exec into a shall with appropriate AWS credentials (including MFA). - ## Running Tests To execute tests on the `deployer`, you will need to install the development requirements and then invoke `pytest` from the root of the repository. diff --git a/deployer/commands/exec/infra_components.py b/deployer/commands/exec/infra_components.py index a63753b2d9..beccd6f0d4 100644 --- a/deployer/commands/exec/infra_components.py +++ b/deployer/commands/exec/infra_components.py @@ -1,6 +1,7 @@ import json import os import subprocess +import tempfile import typer from ruamel.yaml import YAML @@ -22,13 +23,18 @@ def root_homes( cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"), hub_name: str = typer.Argument(..., help="Name of hub to operate on"), - extra_nfs_server: str = typer.Argument( + persist: bool = typer.Option( + False, + "--persist", + help="Do not automatically delete the pod after completing. Useful for long-running processes.", + ), + extra_nfs_server: str = typer.Option( None, help="IP address of an extra NFS server to mount" ), - extra_nfs_base_path: str = typer.Argument( + extra_nfs_base_path: str = typer.Option( None, help="Path of the extra NFS share to mount" ), - extra_nfs_mount_path: str = typer.Argument( + extra_nfs_mount_path: str = typer.Option( None, help="Mount point for the extra NFS share" ), ): @@ -40,12 +46,11 @@ def root_homes( with open(config_file_path) as f: cluster = Cluster(yaml.load(f), config_file_path.parent) - with cluster.auth(): - hubs = cluster.hubs - hub = next((hub for hub in hubs if hub.spec["name"] == hub_name), None) - if not hub: - print_colour("Hub does not exist in {cluster_name} cluster}") - return + hubs = cluster.hubs + hub = next((hub for hub in hubs if hub.spec["name"] == hub_name), None) + if not hub: + print_colour("Hub does not exist in {cluster_name} cluster}") + return server_ip = base_share_name = "" for values_file in hub.spec["helm_chart_values_files"]: @@ -94,6 +99,10 @@ def root_homes( pod = { "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": pod_name, + "namespace": hub_name, + }, "spec": { "terminationGracePeriodSeconds": 1, "automountServiceAccountToken": False, @@ -112,27 +121,42 @@ def root_homes( }, } - cmd = [ + # Command to exec into pod + exec_cmd = [ "kubectl", "-n", hub_name, - "run", - "--rm", # Remove pod when we're done - "-it", # Give us a shell! - "--overrides", - json.dumps(pod), - "--image", - # Use ubuntu image so we get GNU rm and other tools - # Should match what we have in our pod definition - UBUNTU_IMAGE, + "exec", + "-it", pod_name, "--", "/bin/bash", "-l", ] - with cluster.auth(): - subprocess.check_call(cmd) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json") as tmpf: + # Dump the pod spec to a temporary file + json.dump(pod, tmpf) + tmpf.flush() + + with cluster.auth(): + try: + # We are splitting the previous `kubectl run` command into a + # create and exec couplet, because using run meant that the bash + # process we start would be assigned PID 1. This is a 'special' + # PID and killing it is equivalent to sending a shutdown command + # that will cause the pod to restart, killing any processes + # running in it. By using create then exec instead, we will be + # assigned a PID other than 1 and we can safely exit the pod to + # leave long-running processes if required. + # + # Ask api-server to create a pod + subprocess.check_call(["kubectl", "create", "-f", tmpf.name]) + # Exec into pod + subprocess.check_call(exec_cmd) + finally: + if not persist: + delete_pod(pod_name, hub_name) @exec_app.command() @@ -180,7 +204,7 @@ def homes( "-n", hub_name, "run", - "--rm", # Remove pod when we're done + "--rm", # Deletes the pod when the process completes, successfully or otherwise "-it", # Give us a shell! "--overrides", json.dumps(pod), diff --git a/docs/howto/features/storage-quota.md b/docs/howto/features/storage-quota.md index 1409c48372..88454ad5e2 100644 --- a/docs/howto/features/storage-quota.md +++ b/docs/howto/features/storage-quota.md @@ -82,7 +82,7 @@ Here's an example of how to do this: ```bash # Create a throwaway pod with both the existing home directories and the new NFS server mounted -deployer exec root-homes --extra_nfs_server= --extra_nfs_base_path=/ --extra_nfs_mount_path=/new-nfs-volume +deployer exec root-homes --extra-nfs-server= --extra-nfs-base-path=/ --extra-nfs-mount-path=/new-nfs-volume # Copy the existing home directories to the new NFS server while keeping the original permissions rsync -av --info=progress2 /root-homes/ /new-nfs-volume/